[{"id":"gemini-3-1-pro-preview","name":"Gemini 3.1 Pro Preview","org":"Google DeepMind","color":"#64748b","category":"preview","params_b":null,"context_window":1048576,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1160.6,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":35.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":77.08,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":57,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":26.464,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":20.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning from everyday-life context","label":"CL-bench Life","trust":2,"value":16.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench_life","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":51,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":17.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":97.07,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":47.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":11.726,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":14.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":36.78,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing working exploits","label":"ExploitBench","trust":2,"value":26.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"exploitbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":59.649,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":26.829,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":26.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":0.846,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":94.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":22.55,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":46.44,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1444,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1487,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1279,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":53.82,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":384.147,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":8.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":34,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":95.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":18.11,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":58.681,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":79.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":73.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":80.225,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":911.214,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1446.64,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":72.07,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":40,"released_at":null,"release_source":null,"price_in":2.0,"price_out":12.0,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3.1-pro-preview","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-8","name":"Claude Opus 4.8","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":1000000,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1563.83,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":48.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":14.534,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":52,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":20.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":94.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":50.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":58.974,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":28.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":23.51,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":159.91,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":46.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":80,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":56.098,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":42.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":70.905,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":24,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":91.035,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":47.06,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1534,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1464,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1204,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1475,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1279,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":57.51,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":65.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks","label":"OSWorld 2","trust":2,"value":20.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"osworld_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":98.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":31,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":69,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":8.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":53.472,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":64.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":87.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":5787.428,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1558.83,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":82.89,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":39,"released_at":"2026-05-28","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-8","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-5-2","name":"GLM-5.2","org":"Z.ai (Zhipu AI)","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1046.95,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":45.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":22.775,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":22,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":20.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":93.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":43.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":9.524,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":153.04,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":24.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":59.206,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":29.268,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":0.017,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":91.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":45.82,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":51.919334,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":79.654,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":73.73967,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":62.29175,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":76.24167,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":89.78125,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":78.625,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":36.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":19,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":86.389,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":30.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":35,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":78.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":51.157,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":58.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":34.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":55.625,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":8313.778,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1593.25,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":70.12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":35,"released_at":"2026-06-16","release_source":"epoch","price_in":0.007,"price_out":6.8,"cheapest_provider":"openrouter","cheapest_model":"z-ai/glm-5.2","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4-xhigh","name":"GPT-5.4 (xhigh)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":73.95,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":27.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning from everyday-life context","label":"CL-bench Life","trust":2,"value":21.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench_life","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":48,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":23.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":94.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":51.77,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":25.397,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":15.96,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":78.596,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":49,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":37.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":93.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":31.37,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":36.24,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":51.97,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":53.838333,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":77.5415,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":79.31333,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":70.21675,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":82.63367,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":94.148,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":88.1155,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"tool use across 1,000 MCP tasks","label":"MCP Atlas","trust":4,"value":70.6,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/mcp_atlas","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"mcp_atlas","source_label":"Scale Labs MCP Atlas leaderboard","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":341.735,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":23.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":37,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":95.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":56,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":56.597,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":45.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":77.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":32,"released_at":"2026-03-05","release_source":"epoch","price_in":2.5,"price_out":15.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-high","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-5-xhigh","name":"GPT-5.5 (xhigh)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1942.97,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":85,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":63,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":27.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":96,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":67.035,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":34.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":85.263,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":72.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":44.167,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":40.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":54.27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":52.070667,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":82.1495,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":81.57567,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":70.72925,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":87.363,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":95.85725,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":89.65375,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"tool use across 1,000 MCP tasks","label":"MCP Atlas","trust":4,"value":75.3,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/mcp_atlas","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"mcp_atlas","source_label":"Scale Labs MCP Atlas leaderboard","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":23.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":56,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks","label":"OSWorld 2","trust":2,"value":13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"osworld_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":23.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":50,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":55.787,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":63,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1510.15,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":84.91,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":30,"released_at":"2026-04-23","release_source":"epoch","price_in":5.0,"price_out":30.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-5-max","name":"Claude Opus 5 (max)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":65.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":90.42,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":63.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":43,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":29.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":46.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":97.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":73.649,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":45.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":53.38,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":85.614,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":73.171,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":52.012,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":60.833,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":24,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":93.876,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1692,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1489,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":63.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":87.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":59,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks","label":"OSWorld 2","trust":2,"value":31.43,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"osworld_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":98.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":99,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":56.366,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":59.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1686.99,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":91.78,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":28,"released_at":"2026-07-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"aihubmix","cheapest_model":"claude-opus-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"minimax-m3","name":"MiniMax-M3","org":"MiniMax","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":640.02,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":37.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":3.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":78.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":147.9,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":14.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":0.914,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":90.909,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1435,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":33.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":40.656334,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":68.2025,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":76.166336,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":57.50825,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":76.83567,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":76.9475,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":74.48075,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks","label":"OSWorld 2","trust":2,"value":4.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"osworld_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":71.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":18,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"fresh, decontaminated GitHub-issue tasks (resolved %)","label":"SWE-rebench","trust":2,"value":47.2,"score_max":100,"source_url":"https://swe-rebench.com/","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_rebench","source_label":"SWE-rebench leaderboard","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":47.106,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":45.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":55,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":2157.772,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1486.7,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":28,"released_at":"2026-06-01","release_source":"epoch","price_in":0.225,"price_out":0.9,"cheapest_provider":"empiriolabs","cheapest_model":"minimax-m3","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-5-flash","name":"Gemini 3.5 Flash","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":1048576,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":911.02,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":27.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":33.623,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":45,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":13.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":94.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":37.389,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":4.762,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":25.41,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":155.35,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":62.807,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":26.829,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":6.701,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":92.803,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":47.15,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":32,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":95.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":31,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":79.339,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":53.935,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":76.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":66.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":58.125,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":5396.418,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1506.38,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":62.64,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":27,"released_at":"2026-05-19","release_source":"epoch","price_in":1.5,"price_out":9.0,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-3.5-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k2-6","name":"Kimi K2.6","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1092.67,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":3.905,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":20,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":90.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":2.381,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":151.62,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing working exploits","label":"ExploitBench","trust":2,"value":18.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"exploitbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":57.193,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":25.641,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":21.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":0.862,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":90.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1451,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1265,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":37.26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":18,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks","label":"OSWorld 2","trust":2,"value":4.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"osworld_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":96.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":76.653,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":53.472,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":34.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":6204.57,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1508.77,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":55.86,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":27,"released_at":"2026-04-20","release_source":"epoch","price_in":0.38,"price_out":1.6,"cheapest_provider":"clf_ai_gateway","cheapest_model":"kimi-k2.6","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-astra-max","name":"GPT-6 Astra (max)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":2951.32,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":95,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":68.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":74,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":31.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":73.23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":76.19,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":53.26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":93.684,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":97.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":65.508,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":80,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":50.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":95.77,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1788,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1468,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1477,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1284,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":60,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":84,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":41.88,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":56.481,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":75.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1800.28,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":93.26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":26,"released_at":"2026-09-03","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"azure","cheapest_model":"gpt-6-astra","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k2-7-code","name":"Kimi K2.7 Code","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":886.23,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":37.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":10,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":30.531,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":151.15,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":30.06,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":54.035,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":12.195,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":0.853,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":87.879,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":45.656334,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":73.959,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":62.657333,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":56.29175,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":77.90567,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":79.60425,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":82.80775,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":95.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":47.801,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":57.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":36.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":48.75,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":5082.936,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1472.72,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":54.12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":26,"released_at":"2026-06-12","release_source":"epoch","price_in":0.38,"price_out":1.6,"cheapest_provider":"clf_ai_gateway","cheapest_model":"kimi-k2.7-code","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-7-max","name":"Qwen3.7 Max","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1189.4,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":13.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":92.27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":9.524,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":154.58,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":64.561,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":34.146,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":0.407,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":90.909,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":44.02,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":43.586,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":74.219,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":71.786,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":74.0415,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":79.739,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":85.24825,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":83.3365,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":32,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":95.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":77.273,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":49.537,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":70.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":55.767,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1517.12,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":26,"released_at":"2026-05-19","release_source":"epoch","price_in":1.25,"price_out":3.75,"cheapest_provider":"novita","cheapest_model":"qwen/qwen3.7-max","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-8-flash-high","name":"Gemini 3.8 Flash (high)","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1269.72,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":89.166664,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":58,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":18.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":39.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":73.826,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":68.421,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":21.951,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":19.627,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":31.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":44.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":95.391,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1583,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-07","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-06)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1495,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1290,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":38.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":98.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":56.597,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":69.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":76.875,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1567.88,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":84.81,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":7.379,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":24,"released_at":"2026-09-02","release_source":"epoch","price_in":0.75,"price_out":3.75,"cheapest_provider":"gemini","cheapest_model":"gemini-3.8-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-4-argon-high","name":"gemini-4-argon-high","org":"Google","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":2100.1,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":91.25,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":59,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":27.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":52.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":97.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":64.762,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":85.965,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":78.049,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":54.961,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":72.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":53.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":96.402,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1678,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1525,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":63.69,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":58.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":64,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":40.44,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":61.806,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":76,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1679.5,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":20.262,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":24,"released_at":"2026-09-30","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"gemini","cheapest_model":"gemini-4-argon","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-sol-max","name":"GPT-5.6 Sol (max)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":2176.88,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":92.5,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":60,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":59,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":32.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":41.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":95.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":72.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":44.762,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":89.123,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":82.927,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":32.202,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":56.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":30.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":93.497,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":58.43,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":58,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks","label":"OSWorld 2","trust":2,"value":27.34,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"osworld_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":31.01,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":83,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":57.06,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":69.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":86.97,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":24,"released_at":"2026-07-09","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"replicate","cheapest_model":"openai/gpt-5.6-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k3-max","name":"Kimi K3 (max)","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1524.47,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":60.416668,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":23.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":91.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":68.514,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":72.183,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":39.024,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":25.865,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":34.167,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":39.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":93.119,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1655,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1488,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":52.74,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":54.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":97.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":59.491,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":60.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":50.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1674.26,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":82.57,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":24,"released_at":"2026-07-16","release_source":"epoch","price_in":0.192,"price_out":16.0,"cheapest_provider":"openrouter","cheapest_model":"moonshotai/kimi-k3","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-1-max","name":"Claude Fable 5.1 (max)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":90,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":48,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":29.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":51.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":57.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":90.175,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":87.805,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":56.291,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":70,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":49.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1745,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1513,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1501,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1288,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":93.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":58,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":44.65,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":63.079,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":70.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1758.05,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":92.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":23,"released_at":"2026-09-01","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5-1","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-5-5-max","name":"Claude Opus 5.5 (max)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":73.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":91.666664,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":31.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":57.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":98.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":71.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":91.228,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":95,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":62.331,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":83.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":48.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":90.593,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1814,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":68.23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":95.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":71,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":43.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":66.898,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":72.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1820.31,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":23,"released_at":"2026-09-22","release_source":"curated","price_in":4.0,"price_out":20.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-flash-preview","name":"Gemini 3 Flash Preview","org":"Google DeepMind","color":"#64748b","category":"preview","params_b":null,"context_window":1048576,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1367.2,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":48.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":40,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":89.07,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":49.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":51.228,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":17.073,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":10,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":89.394,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":9.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":43.13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"tool use across 1,000 MCP tasks","label":"MCP Atlas","trust":4,"value":62,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/mcp_atlas","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"mcp_atlas","source_label":"Scale Labs MCP Atlas leaderboard","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":95.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":15,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":75.413,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":61.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":66.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":64.27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":3634.716,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1383.12,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":61.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":23,"released_at":null,"release_source":null,"price_in":0.5,"price_out":3.0,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3-flash-preview","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-6-flash","name":"Gemini 3.6 Flash","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":1048576,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":715.52,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":46.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":31.236,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":42,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":10.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":95.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":46.681,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":155.24,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":34.37,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":58.947,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":21.951,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":23.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":94.129,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":44.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":23.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":30,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":94.167,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":53.356,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":66.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1536.95,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":56.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":23,"released_at":"2026-07-21","release_source":"epoch","price_in":0.75,"price_out":3.75,"cheapest_provider":"gemini","cheapest_model":"gemini-3.6-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-1-sol-max","name":"GPT-6.1 Sol (max)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":2743.75,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":60,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":94.166664,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":68,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":31.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":96.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":54.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":93.684,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":80,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":51.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":95.391,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1757,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1483,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1291,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":62.78,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":61.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":80,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":38.79,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":54.167,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":73.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1758.72,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":23,"released_at":"2026-09-29","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.1-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-max","name":"Claude Fable 5 (max)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":89.166664,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":28.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":98.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":69.725,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":39.524,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":87.018,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":90.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":46.964,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":35.833,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":29.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":85.859,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":60.34,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":70,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":52,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":99.722,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":37.27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":95,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":60.995,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":81.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":4966.64,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":91.94,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":22,"released_at":"2026-06-09","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-7-flash-high","name":"Gemini 3.7 Flash (high)","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":904.3,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":84.583336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":66,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":14.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":96.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":65.265,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":71.579,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":36.585,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":20.261,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":26.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":23.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":94.823,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1592,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-07","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-06)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1488,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1296,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":50.38,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":41.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":37,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":97.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":57.176,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":69.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1587.26,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":22,"released_at":"2026-08-13","release_source":"epoch","price_in":0.75,"price_out":3.75,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3.7-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-high","name":"GPT-5 (high)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1162.45,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":9.86,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":12.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":90.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":48.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":12.698,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":55.439,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":21.951,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":86.174,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":6.86,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":25.32,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":39.99,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":203.013,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":4.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":91.389,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":18,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":73.554,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":56.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":50.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":60.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":22,"released_at":"2025-08-07","release_source":"curated","price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-5-high","name":"GPT-5.5 (high)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":83.33,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"learning from everyday-life context","label":"CL-bench Life","trust":2,"value":22.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench_life","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":25.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":54,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":64.381,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1484,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1481,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1281,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":46.767666,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":80.037,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":80.443,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":71.375,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":87.76,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":95.19,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":89.673,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":10,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":52,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":56.134,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":88.125,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1486.98,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":83.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":21,"released_at":"2026-04-23","release_source":"epoch","price_in":5.0,"price_out":30.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-sol-max","name":"GPT-6 Sol (max)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":2462,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":54.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":89.583336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":56,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":30.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":97.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":53.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":49.27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":89.825,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":90,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":58.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":26.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":94.318,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1687,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":59.08,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":48.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":56,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":57.639,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":60.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1689.33,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":21,"released_at":"2026-09-22","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-7-xhigh","name":"Grok 4.7 (xhigh)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1545.85,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":61.38889,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":41,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":17.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":46.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":96,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":52.982,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":17.073,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":29.505,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":20.833,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":48.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":92.74,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1638,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":49.38,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":54.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":29,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":98.056,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":57.407,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":56,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1637.52,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":8.453,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":21,"released_at":"2026-09-21","release_source":"curated","price_in":2.0,"price_out":6.0,"cheapest_provider":"azure_ai","cheapest_model":"grok-4.7","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k2-5","name":"Kimi K2.5","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":821.65,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":11.81,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":19.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning from everyday-life context","label":"CL-bench Life","trust":2,"value":13.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench_life","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":3.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":3.38,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":148.8,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":86.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":87.6,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":24.37,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":87.1,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"computer-use tasks (v1)","label":"OSWorld","trust":2,"value":63.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"os_world","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Multilingual","trust":4,"value":67.3,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multilingual","source_label":"SWE-bench Multilingual leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":70.8,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":48.958,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":46.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":34.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":43.202,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":1198.458,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1436.37,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":45.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":21,"released_at":"2026-01-27","release_source":"epoch","price_in":0.45,"price_out":2.25,"cheapest_provider":"deepinfra","cheapest_model":"moonshotai/Kimi-K2.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-high","name":"Claude Fable 5 (high)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":2041.3,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":87.5,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":45,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":96.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":68.605,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":39.28,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":83.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1625,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-07","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-06)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1496,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1230,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1504,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1309,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":61.15,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":63.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"fresh, decontaminated GitHub-issue tasks (resolved %)","label":"SWE-rebench","trust":2,"value":64.5,"score_max":100,"source_url":"https://swe-rebench.com/","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_rebench","source_label":"SWE-rebench leaderboard","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":95,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":5680.258,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1627.98,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":87.85,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":20,"released_at":"2026-06-09","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-7-max","name":"Claude Opus 4.7 (max)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":49.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":41,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":94.65,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":19.048,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":70.175,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":31.707,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":33.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":21,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":86.364,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":52.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"tool use across 1,000 MCP tasks","label":"MCP Atlas","trust":4,"value":79.1,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/mcp_atlas","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"mcp_atlas","source_label":"Scale Labs MCP Atlas leaderboard","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":43.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":28,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks","label":"OSWorld 2","trust":2,"value":18.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"osworld_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":86.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":54,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":83.471,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":54.514,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":75.49,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":20,"released_at":"2026-04-16","release_source":"epoch","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-7","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-5-3-max","name":"GLM-5.3 (max)","org":"Z.ai (Zhipu AI)","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":37,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":19.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":42.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":68.958,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":40.14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":68.772,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":29.268,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":30.182,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":36.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":90.909,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1623,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-07","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-06)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1478,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":91.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":34.52,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":49,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":59.028,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":41,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1614.45,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":75.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":20,"released_at":"2026-08-14","release_source":"epoch","price_in":0.0699,"price_out":1.4,"cheapest_provider":"openrouter","cheapest_model":"z-ai/glm-5.3","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-5-high","name":"Grok 4.5 (high)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1309.28,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":52.64,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":35,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":15.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":96.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":53.761,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":57.193,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":24.39,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":22.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":93.434,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":45.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":32.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":97.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":22.62,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":31,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"fresh, decontaminated GitHub-issue tasks (resolved %)","label":"SWE-rebench","trust":2,"value":63.8,"score_max":100,"source_url":"https://swe-rebench.com/","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_rebench","source_label":"SWE-rebench leaderboard","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":54.977,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":48.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":74.375,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":20,"released_at":"2026-07-08","release_source":"epoch","price_in":2.0,"price_out":6.0,"cheapest_provider":"github_copilot","cheapest_model":"grok-4.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-6-xhigh","name":"Grok 4.6 (xhigh)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1508.08,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":67.083336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":19.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":41.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":97.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":66.741,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":30.476,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":65.965,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":31.707,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":25.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":40,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":17.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":93.182,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":48.45,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":41.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":34,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":99.167,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":53.009,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":48.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":20,"released_at":"2026-08-12","release_source":"curated","price_in":2.0,"price_out":6.0,"cheapest_provider":"aihubmix","cheapest_model":"grok-4.6","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-pro-preview","name":"Gemini 3 Pro Preview","org":"Google DeepMind","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1176.75,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":58.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":15.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":6.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":18.24,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":92.614,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":18.63,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":37.52,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"tool use across 1,000 MCP tasks","label":"MCP Atlas","trust":4,"value":70.3,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/mcp_atlas","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"mcp_atlas","source_label":"Scale Labs MCP Atlas leaderboard","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":224.326,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":91.389,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":20,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":1.25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":74.2,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":76.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":69.426,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":5478.158,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1438.77,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":69.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":19,"released_at":null,"release_source":null,"price_in":2.0,"price_out":12.0,"cheapest_provider":"replicate","cheapest_model":"google/gemini-3-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4-mini-xhigh","name":"GPT-5.4 mini (xhigh)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":18.9,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":10,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":80,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":51.228,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":9.756,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":86.869,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":40.79,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":41.666668,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":71.624,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":70.78567,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":59.804,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":70.95167,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":78.462,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":71.32375,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"tool use across 1,000 MCP tasks","label":"MCP Atlas","trust":4,"value":56.7,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/mcp_atlas","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"mcp_atlas","source_label":"Scale Labs MCP Atlas leaderboard","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":88.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":21,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":52.083,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":19,"released_at":"2026-03-17","release_source":"epoch","price_in":0.75,"price_out":4.5,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-luna-max","name":"GPT-5.6 Luna (max)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1667.4,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":59.54,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":45.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":43,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":20.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":35.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":88.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":67.188,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":82.105,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":60.976,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":42.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":91.604,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":48.54,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":21.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":21,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":98.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":60,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":53.588,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":41,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":19,"released_at":"2026-07-09","release_source":"epoch","price_in":0.2,"price_out":1.2,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.6-luna","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-terra-max","name":"GPT-5.6 Terra (max)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1951.42,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":58.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":83.9,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":53.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":30,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":41.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":93.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":69.623,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":85.965,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":70.732,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":54.167,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":93.308,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":54.95,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":41.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":35,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":99.722,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":54.977,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":43.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":19,"released_at":"2026-07-09","release_source":"epoch","price_in":2.0,"price_out":12.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.6-terra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-luna-max","name":"GPT-6 Luna (max)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":44.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":59.305557,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":19.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":90.13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":42.42,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":78.947,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":56.098,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":44.167,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":90.467,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1579,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":44.49,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":26.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":98.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":54.63,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":41.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1581.95,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":19,"released_at":"2026-09-22","release_source":"epoch","price_in":0.1,"price_out":0.5,"cheapest_provider":"azure","cheapest_model":"gpt-6-luna","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-haiku-5-5-max","name":"Claude Haiku 5.5 (max)","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":56.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":55,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":18.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":48.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":94.99,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":43.81,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":46.36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":75.088,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":46.341,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":43.769,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":47.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":89.646,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":51.06,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":78.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":30,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":89,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":54.977,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":23.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":18,"released_at":"2026-10-07","release_source":"epoch","price_in":0.1,"price_out":0.5,"cheapest_provider":"anthropic","cheapest_model":"claude-haiku-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-5-5-max","name":"Claude Sonnet 5.5 (max)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":75.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":59,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":31.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":55.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":97.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":88.772,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":80.488,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":61.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":75,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":51.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":95.581,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":68.26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":97.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":65,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":60.995,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":46.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":18,"released_at":"2026-09-28","release_source":"curated","price_in":2.0,"price_out":10.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-5-flash-lite","name":"Gemini 3.5 Flash-Lite","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":1048576,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":765.27,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":29.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":21,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":83.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":145.65,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":25.965,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":10,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":83.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1264,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":37.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":19,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":71.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":41.319,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1446.62,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":39,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":18,"released_at":"2026-07-21","release_source":"epoch","price_in":0.3,"price_out":2.5,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3.5-flash-lite","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-5-1","name":"GLM-5.1","org":"Z.ai (Zhipu AI)","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":887.1,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":40.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":20,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":4.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":150.73,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing working exploits","label":"ExploitBench","trust":2,"value":18.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"exploitbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":36.842,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":86.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":93.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":22.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":74.17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":44.792,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":55.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":34,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":5634.406,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1508.29,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":57.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":18,"released_at":"2026-04-07","release_source":"epoch","price_in":0.825,"price_out":3.301,"cheapest_provider":"empiriolabs","cheapest_model":"glm-5-1","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4-1","name":"GPT-4.1","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":1047576,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":558.1,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0.42,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":52.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":11,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":68.27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":2.17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":136.18,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":63.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":5.965,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":66.919,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":5.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":25.61,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":38.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Multimodal","trust":4,"value":31.14,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multimodal","source_label":"SWE-bench Multimodal leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":39.58,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":31.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":39.04,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":18,"released_at":"2025-04-14","release_source":"curated","price_in":2.0,"price_out":8.0,"cheapest_provider":"azure","cheapest_model":"gpt-4.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-6-27b","name":"Qwen3.6 27B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":78.13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":146.78,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":35.088,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":87.8,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":34.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":39.292667,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":71.785,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":70.42667,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":53.229,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":63.304333,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":79.8685,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":70.28375,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":86.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":91.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"fresh, decontaminated GitHub-issue tasks (resolved %)","label":"SWE-rebench","trust":2,"value":31.2,"score_max":100,"source_url":"https://swe-rebench.com/","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_rebench","source_label":"SWE-rebench leaderboard","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":37.269,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":18,"released_at":"2026-04-22","release_source":"epoch","price_in":0.15,"price_out":0.5,"cheapest_provider":"libertai","cheapest_model":"qwen3.6-27b","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4","name":"Claude Opus 4","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.27,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":38,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":81.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":46.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":5.57,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":142.88,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":61.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":76.263,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":6.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":10.72,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":37.43,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":100.366,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":64.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":70.661,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":58.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":43.72,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":17,"released_at":"2025-05-22","release_source":"curated","price_in":15.0,"price_out":75.0,"cheapest_provider":"gmi","cheapest_model":"anthropic/claude-opus-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-6-max","name":"Claude Opus 4.6 (max)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":46.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":21,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":91.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":12.698,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":7.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":65.965,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":26.829,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":28.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":88.384,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":34.44,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":55.76,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"tool use across 1,000 MCP tasks","label":"MCP Atlas","trust":4,"value":76.8,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/mcp_atlas","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"mcp_atlas","source_label":"Scale Labs MCP Atlas leaderboard","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":33.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":91.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":50,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":17,"released_at":"2026-02-05","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-1-flash-max","name":"deepseek-v4.1-flash-max","org":"DeepSeek","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1092.35,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":72.916664,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":48,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":14.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":67.368,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":26.829,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":34.167,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":19.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":89.836,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1620,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-07","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-06)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1474,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":64.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":43,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":98.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":51.852,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":42.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1613.95,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":17,"released_at":"2026-09-09","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-4-carbon-high","name":"Gemini 4 Carbon (high)","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":2374.7,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":92.083336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":78,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":60.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":87.619,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":90.526,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":95.122,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":61.199,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":77.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":56.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":96.528,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":73,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":40.98,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":64.236,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":77.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":17,"released_at":"2026-10-28","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"gemini","cheapest_model":"gemini-4-carbon","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark-1-3-max","name":"Muse Spark 1.3 (max)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":47.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":37,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":24.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":41.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":74.035,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":46.341,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":25.802,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1656,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1471,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1494,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1290,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":54.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":58,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":58.796,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1651.94,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":17,"released_at":"2026-09-02","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"empiriolabs","cheapest_model":"muse-spark-1-3","price_observed_at":"2026-11-18T15:47Z"},{"id":"o3-high","name":"o3 (high)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":933.55,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":6.53,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":17.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":1.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":84.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":11.91,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":33.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":81.818,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":8.82,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":20.32,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":39.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":29,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":83.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":53.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":49.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":52.42,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":17,"released_at":"2025-04-16","release_source":"curated","price_in":2.0,"price_out":8.0,"cheapest_provider":"azure","cheapest_model":"o3","price_observed_at":"2026-11-18T15:47Z"},{"id":"o4-mini-high","name":"o4-mini (high)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":826.17,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":6.11,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":77.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":9.21,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":36.14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":4.878,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":79.609,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":3.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":18.08,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":26.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":81.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":38.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":18.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":52.56,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":17,"released_at":"2025-04-16","release_source":"curated","price_in":1.0,"price_out":4.0,"cheapest_provider":"replicate","cheapest_model":"openai/o4-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-2-max","name":"Claude Fable 5.2 (max)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":92.5,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":79,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":35.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":60,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":90.476,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":95.088,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":69.94,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":90,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":93.876,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":90,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":99.722,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":44.64,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":67.708,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":94,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":16,"released_at":"2026-11-09","release_source":"epoch","price_in":8.0,"price_out":40.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5-2","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-5-high","name":"Claude Opus 5 (high)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":2164.57,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":88.33,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":28.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":44.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":97.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":72.829,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1658,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1516,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1490,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1289,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":63.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks","label":"OSWorld 2","trust":2,"value":28.98,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"osworld_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"fresh, decontaminated GitHub-issue tasks (resolved %)","label":"SWE-rebench","trust":2,"value":63.4,"score_max":100,"source_url":"https://swe-rebench.com/","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_rebench","source_label":"SWE-rebench leaderboard","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":54.282,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1660.45,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":91.59,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":16,"released_at":"2026-07-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"aihubmix","cheapest_model":"claude-opus-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4-1-mini","name":"GPT-4.1 mini","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":1047576,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":32.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":68.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":134.27,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":44.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":6.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":65.846,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":21.12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":44.722,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":23.94,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":40.394,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":12.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":37.61,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":16,"released_at":"2025-04-14","release_source":"curated","price_in":0.4,"price_out":1.6,"cheapest_provider":"azure","cheapest_model":"gpt-4.1-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-2-xhigh","name":"GPT-5.2 (xhigh)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":52.91,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":48,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":90.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":23.016,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":67.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":31.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":38.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":91.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":43.91,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"tool use across 1,000 MCP tasks","label":"MCP Atlas","trust":4,"value":67.6,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/mcp_atlas","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"mcp_atlas","source_label":"Scale Labs MCP Atlas leaderboard","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":32.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":96.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":15,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":37.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":72.19,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":5.008,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":16,"released_at":"2025-12-11","release_source":"curated","price_in":1.75,"price_out":14.0,"cheapest_provider":"azure","cheapest_model":"gpt-5.2","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-5-max","name":"Claude Sonnet 5 (max)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":45,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":16.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":34.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":92.48,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":53.846,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":65.614,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":29.268,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":80.303,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":49.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":63.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":35,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":80,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":77,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":53.588,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":33.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":15,"released_at":"2026-06-30","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"aihubmix","cheapest_model":"claude-sonnet-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-pro","name":"gemini-2.5-pro","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":1048576,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":82.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":41.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":24.561,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":85.29,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":3.92,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1421,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":34.77,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":84.722,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":53.6,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":46.296,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":32.64,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":573.642,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1225.73,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":15,"released_at":"2025-03-25","release_source":"curated","price_in":1.25,"price_out":10.0,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-5-3-flash-max","name":"GLM-5.3-Flash (max)","org":"Z.ai (Zhipu AI)","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":65.833336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":21,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":36.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":63.393,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":31.83,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":55.789,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":17.073,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":18.125,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":90.152,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":93.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":36.79,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":21,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":52.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":15,"released_at":"2026-08-20","release_source":"epoch","price_in":0.06,"price_out":0.2,"cheapest_provider":"clf_ai_gateway","cheapest_model":"glm-5.3-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-mini-high","name":"GPT-5 mini (high)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":799.77,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":4.44,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":24,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":80.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":46.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":12.195,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":75,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":34.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":86.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":39.005,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":21.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":52.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":15,"released_at":"2025-08-07","release_source":"curated","price_in":0.25,"price_out":2.0,"cheapest_provider":"azure","cheapest_model":"gpt-5-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-1-high","name":"GPT-5.1 (high)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1192.15,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":23.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning from everyday-life context","label":"CL-bench Life","trust":2,"value":17.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench_life","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":42,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":90.13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":87.626,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":13.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1251,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":43.89,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"tool use across 1,000 MCP tasks","label":"MCP Atlas","trust":4,"value":50.1,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/mcp_atlas","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"mcp_atlas","source_label":"Scale Labs MCP Atlas leaderboard","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":88.611,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":67.975,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":53.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":48,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":60.77,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":15,"released_at":"2025-11-12","release_source":"curated","price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"global/gpt-5.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-sol-xhigh","name":"GPT-5.6 Sol (xhigh)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":90,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":28.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":37.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":95.73,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":70.732,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1483,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1257,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1484,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1285,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":58.49,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":57.06,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":64.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":93.125,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1617.32,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":14.992,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":15,"released_at":"2026-07-09","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"replicate","cheapest_model":"openai/gpt-5.6-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-6-high","name":"Grok 4.6 (high)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":65.138885,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":17.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":40.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":65.188,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":94.003,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1620,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-07","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-06)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1452,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1263,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":97.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":56.481,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":49.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1617.64,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":67.29,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":15,"released_at":"2026-08-12","release_source":"curated","price_in":2.0,"price_out":6.0,"cheapest_provider":"aihubmix","cheapest_model":"grok-4.6","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-3","name":"grok-4.3","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":944.17,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1165,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":18.535334,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":69.9325,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":55.772667,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":62.75,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":73.58,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":84.339,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":70.822,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":35.262,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1357.17,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":49.89,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":15,"released_at":"2026-04-17","release_source":"epoch","price_in":1.25,"price_out":2.5,"cheapest_provider":"azure_ai","cheapest_model":"grok-4.3","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark-1-1","name":"Muse Spark 1.1","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":31.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":15.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":53.319,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":154.92,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":7.917,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":15,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1542,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1465,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1491,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1281,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":39,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":58.796,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":57.789,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":6520.475,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1541.55,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":15,"released_at":"2026-07-09","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"empiriolabs","cheapest_model":"muse-spark-1-1","price_observed_at":"2026-11-18T15:47Z"},{"id":"o3-mini-high","name":"o3-mini (high)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":2.99,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":15,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":68.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":18.596,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":77.02,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":1.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":18.97,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":76.944,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":42.824,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":22.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":15.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":43.75,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":15,"released_at":"2025-01-31","release_source":"curated","price_in":1.1,"price_out":4.4,"cheapest_provider":"azure","cheapest_model":"o3-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen-3-6-plus-2026-04-02","name":"Qwen 3.6 Plus (2026-04-02)","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":670.15,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":20.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":2.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":81.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":38.246,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":88.384,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":33.09,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":93.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":57.851,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":40.741,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":44.144,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":5114.872,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1460.53,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":15,"released_at":"2026-04-02","release_source":"name","price_in":0.4388,"price_out":2.633,"cheapest_provider":"kyma","cheapest_model":"qwen-3.6-plus","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-flash-0731-max","name":"DeepSeek V4 Flash 0731 (max)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1306.08,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":61.38889,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":32,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":16.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":90.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":57.544,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":24.39,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":91.035,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":41.73,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":34,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":94.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":49.884,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":33.634,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":62.96,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":14,"released_at":"2026-07-31","release_source":"epoch","price_in":0.06,"price_out":0.18,"cheapest_provider":"deepinfra","cheapest_model":"deepseek-ai/DeepSeek-V4-Flash-0731","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-1-flash-lite","name":"Gemini 3.1 Flash-Lite","org":"Google","color":"#64748b","category":"flash","params_b":null,"context_window":1048576,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":797.73,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":1.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":76.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":37.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":3.04,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":145.03,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":27.719,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":81.818,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":8.64,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":35.01,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":80,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":43.403,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1254.27,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":52.19,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":14,"released_at":"2026-03-03","release_source":"epoch","price_in":0.25,"price_out":1.5,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-3.1-flash-lite","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-5","name":"GLM-5","org":"Z.ai (Zhipu AI)","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":765.62,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":4.86,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":18.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":146.54,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":86,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":86,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":80,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Multilingual","trust":4,"value":69.7,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multilingual","source_label":"SWE-bench Multilingual leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":72.8,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":53.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":52.416,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":4432.122,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1435.91,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":48.17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":14,"released_at":"2026-02-12","release_source":"curated","price_in":0.6,"price_out":2.08,"cheapest_provider":"deepinfra","cheapest_model":"zai-org/GLM-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-2-high","name":"GPT-5.2 (high)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1293.55,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":43.33,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":18.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":45,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":88.194,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":27.45,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1405,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":352.249,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":96.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":73.76,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":45.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":34.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1480,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":14,"released_at":"2025-12-11","release_source":"curated","price_in":1.75,"price_out":14.0,"cheapest_provider":"azure","cheapest_model":"gpt-5.2","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark-1-2-xhigh","name":"Muse Spark 1.2 (xhigh)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":17.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":94.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":54.867,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":11.97,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1532,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1494,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1293,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":48.35,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":57.407,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":60.302,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1534.38,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":60.38,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":14,"released_at":"2026-08-05","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"empiriolabs","cheapest_model":"muse-spark-1-2","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-6-35b-a3b","name":"Qwen3.6 35B-A3B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":73.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":20.351,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":86,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":29.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":85.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":22,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":86.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"fresh, decontaminated GitHub-issue tasks (resolved %)","label":"SWE-rebench","trust":2,"value":24.7,"score_max":100,"source_url":"https://swe-rebench.com/","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_rebench","source_label":"SWE-rebench leaderboard","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":36.574,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":44.375,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":24.625,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1250.15,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":34.49,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":14,"released_at":"2026-04-14","release_source":"epoch","price_in":0.05,"price_out":0.4,"cheapest_provider":"sail","cheapest_model":"Qwen/Qwen3.6-35B-A3B","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-5-unknown-thinking","name":"Claude Sonnet 4.5 (unknown thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":1.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":60,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":83.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":5.997,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":14.71,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":13.72,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":38.76,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":19,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":2.08,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":45.718,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":54.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":46.517,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":3838.74,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2025-09-29","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-5-high","name":"Claude Sonnet 5 (high)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1463.12,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":1.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":30.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":84.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":48.23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1539,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1466,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1264,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":50.02,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"fresh, decontaminated GitHub-issue tasks (resolved %)","label":"SWE-rebench","trust":2,"value":56.8,"score_max":100,"source_url":"https://swe-rebench.com/","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_rebench","source_label":"SWE-rebench leaderboard","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":54.282,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1536.72,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":68.98,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2026-06-30","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"aihubmix","cheapest_model":"claude-sonnet-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-5-5-xhigh","name":"Claude Sonnet 5.5 (xhigh)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":83.888885,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":31.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":53.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":96,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":52.09,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":46.12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1773,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1471,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1268,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":66.32,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":57.292,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1786.26,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":19.94,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2026-09-28","release_source":"curated","price_in":2.0,"price_out":10.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-opus-20240229","name":"claude-3-opus-20240229","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":68.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"Aider official benchmark","label":"Aider Refactoring","trust":4,"value":72.3,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/refactor.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_refactor","source_label":"Aider Aider Refactoring leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":10,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":61.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":0.82,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":47.159,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":3.952,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":4.722,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":23.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":12.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":19.22,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2024-03-04","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-v4-pro-max","name":"DeepSeek v4 Pro (max)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":19,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":12.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":90.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":45.263,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":2.439,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":89.646,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":41.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":96.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":77.64,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":50,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":46.994,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":48.61,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2026-04-24","release_source":"epoch","price_in":0.528,"price_out":1.584,"cheapest_provider":"clf_ai_gateway","cheapest_model":"deepseek-v4-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-pro-0813-max","name":"DeepSeek V4 Pro 0813 (max)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1403.15,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":61.25,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":46,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":18,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":64.561,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":26.829,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":91.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":33.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":43,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":98.611,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":51.042,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":52.906,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":66.22,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2026-08-13","release_source":"epoch","price_in":0.111,"price_out":4.44,"cheapest_provider":"openrouter","cheapest_model":"deepseek/deepseek-v4-pro-0813","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-1-flash-1017-max","name":"DeepSeek V4.1 Flash 1017 (max)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1622,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":69.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":78.05556,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":46,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":24,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":47.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":71.228,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":46.341,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":93.182,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":49,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":99.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":55.903,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":42.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2026-10-17","release_source":"epoch","price_in":0.036,"price_out":0.5,"cheapest_provider":"openrouter","cheapest_model":"deepseek/deepseek-v4.1-flash-1017","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-r1","name":"DeepSeek-R1","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":64000,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.3,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":56.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":34.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":1.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":138.63,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":69.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":71.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":26.933,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":84,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":53.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":38.31,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":30.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":36.49,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2025-01-20","release_source":"curated","price_in":0.4,"price_out":0.4,"cheapest_provider":"hyperbolic","cheapest_model":"deepseek-ai/DeepSeek-R1","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-4-7","name":"GLM-4.7","org":"Z.ai (Zhipu AI)","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":399.48,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":21,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":1.71,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":143.36,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":85.7,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":83.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":45.139,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":47.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":32.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":33.39,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":2376.816,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1434.7,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2025-12-22","release_source":"epoch","price_in":0.4,"price_out":1.75,"cheapest_provider":"deepinfra","cheapest_model":"zai-org/GLM-4.7","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4-1-nano","name":"GPT-4.1 nano","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":1047576,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":8.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":52.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":126.81,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":48.927,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":5.46,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":28.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":25.926,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":18.98,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2025-04-14","release_source":"curated","price_in":0.1,"price_out":0.4,"cheapest_provider":"azure","cheapest_model":"gpt-4.1-nano","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-nano-high","name":"GPT-5 nano (high)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":718.67,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":2.61,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":22,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":62.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":20,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":2.439,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":69.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":7.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":81.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":11.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":38.06,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2025-08-07","release_source":"curated","price_in":0.05,"price_out":0.4,"cheapest_provider":"azure","cheapest_model":"gpt-5-nano","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4-high","name":"GPT-5.4 (high)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1607,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":67.5,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"learning from everyday-life context","label":"CL-bench Life","trust":2,"value":19.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench_life","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":42,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":89.899,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":25.49,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1475,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1284,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":15.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":97.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":17.76,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":76.86,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1462.99,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2026-03-05","release_source":"epoch","price_in":2.5,"price_out":15.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-high","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4-nano-xhigh","name":"GPT-5.4 nano (xhigh)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":5.69,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":45,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":9.253,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":80.27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":36.86,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":46.767666,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":70.8385,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":67.64267,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":67.20475,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":62.507,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":90.977,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":81.097,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":47.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2026-03-17","release_source":"epoch","price_in":0.2,"price_out":1.25,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-nano","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-terra-xhigh","name":"GPT-5.6 Terra (xhigh)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":74.17,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":27.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":33.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":92.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":60.177,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1472,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1268,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":52.23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":74,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":52.315,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":48.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":83.75,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1521.25,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2026-07-09","release_source":"epoch","price_in":2.0,"price_out":12.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.6-terra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-1-terra-max","name":"GPT-6.1 Terra (max)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":2608.82,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":68.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":28,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":78.095,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":90.877,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":92.683,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":79.167,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":94.318,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":76.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":74,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":55.093,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":57.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2026-10-26","release_source":"epoch","price_in":1.0,"price_out":5.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.1-terra","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-large-4-preview-high","name":"Mistral Large 4 Preview (high)","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":51.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":39.43,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":58.596,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":26.829,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":20.833,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":32.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":88.068,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":40,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":98.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":20,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":4.196,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":null,"release_source":null,"price_in":0.7,"price_out":2.1,"cheapest_provider":"azure_ai","cheapest_model":"mistral-large-4-latest","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark-1-3-xhigh","name":"Muse Spark 1.3 (xhigh)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1688.35,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":34,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":37.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":74.386,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":41.463,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":27.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1626,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":14.141,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":99.167,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":59.722,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1623.12,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":6.655,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2026-09-02","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"empiriolabs","cheapest_model":"muse-spark-1-3","price_observed_at":"2026-11-18T15:47Z"},{"id":"nemotron-3-ultra","name":"nemotron-3-ultra","org":"Nvidia","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":22.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":3.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":90.13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":147,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":13.61,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":85.354,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":36.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":20,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":86.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":40.278,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":41.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":43.45,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2026-06-04","release_source":"epoch","price_in":0.5,"price_out":2.2,"cheapest_provider":"openrouter","cheapest_model":"nvidia/nemotron-3-ultra-550b-a55b","price_observed_at":"2026-11-18T15:47Z"},{"id":"o3-medium","name":"o3 (medium)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":2.98,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":37,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":45.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":13.09,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":88.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":29.825,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":80.808,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":19.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":91.271,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks (v1)","label":"OSWorld","trust":2,"value":23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"os_world","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":84.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":62.319,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2025-04-16","release_source":"curated","price_in":2.0,"price_out":8.0,"cheapest_provider":"azure","cheapest_model":"o3","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-7-plus","name":"Qwen3.7 Plus","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":9.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":84,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":147.95,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":10.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":34.386,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":87.879,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1444,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1265,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":37.58,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks","label":"OSWorld 2","trust":2,"value":2.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"osworld_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":93.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":46.065,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2026-06-02","release_source":"epoch","price_in":0.276,"price_out":1.101,"cheapest_provider":"replicate","cheapest_model":"qwen/qwen3-7-plus","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-8-max-xhigh","name":"Qwen3.8 Max (xhigh)","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":46,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":92,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":57.461,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":74.737,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":46.341,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":15.809,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":42.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":92.677,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":46.24,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":52.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":38,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":99.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":45.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":13,"released_at":"2026-08-02","release_source":"epoch","price_in":1.65,"price_out":4.951,"cheapest_provider":"deepinfra","cheapest_model":"Qwen/Qwen3.8-Max","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-6-unknown-thinking","name":"Claude Opus 4.6 (unknown thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":20.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning from everyday-life context","label":"CL-bench Life","trust":2,"value":13.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench_life","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":26.64,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":44.116,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":33.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":718.807,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":4.17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":79.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":8017.594,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1556.29,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":77.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":12,"released_at":"2026-02-05","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-5-no-thinking","name":"Claude Sonnet 4.5 (no thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":55,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":2.381,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":73.737,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":14.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks (v1)","label":"OSWorld","trust":2,"value":62.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"os_world","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":35.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":71.281,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":23.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":42.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1385.49,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":46.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":12,"released_at":"2025-09-29","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-6-max","name":"Claude Sonnet 4.6 (max)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":58.33,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":3.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":89.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":18,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":78.788,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":46.51,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks","label":"OSWorld 2","trust":2,"value":8.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"osworld_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":71.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":45,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":46.759,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":32.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":12,"released_at":"2026-02-17","release_source":"epoch","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-7-sonnet-20250219","name":"claude-3-7-sonnet-20250219","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":64.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":20,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":2.26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":50,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":66.035,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":3.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":60.389,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks (v1)","label":"OSWorld","trust":2,"value":35.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"os_world","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":21.944,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":60.95,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":44.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"work tasks in a simulated software company","label":"TheAgentCompany","trust":2,"value":30.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"agent_company","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":12,"released_at":"2025-02-24","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemma-4-31b-it","name":"Gemma 4 31B IT","org":"Google DeepMind","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":925.5,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":1.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":82.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":144.12,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":75.758,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":39.25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":73.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":45.486,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":10.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":30.625,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1364.13,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":52.26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":12,"released_at":"2026-04-02","release_source":"epoch","price_in":0.1,"price_out":0.34,"cheapest_provider":"wandb","cheapest_model":"google/gemma-4-31B-it","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4o-mini-2024-07-18","name":"gpt-4o-mini-2024-07-18","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":128000,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":3.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":17.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":54.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":0.702,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":37.721,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":10.37,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":6.944,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":10.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":8.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":11.76,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":12,"released_at":"2024-07-18","release_source":"name","price_in":0.15,"price_out":0.6,"cheapest_provider":"azure","cheapest_model":"gpt-4o-mini-2024-07-18","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-2-codex","name":"GPT-5.2 Codex","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":272000,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1299.9,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":49.394,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":83.6195,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":78.204,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":66.44575,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":73.67834,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":88.77375,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":77.7115,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Multilingual","trust":2,"value":66.3,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_multilingual","source_label":"SWE-bench Multilingual leaderboard (unverified)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":72.8,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":66.517,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1338.7,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":12,"released_at":"2025-12-18","release_source":"epoch","price_in":1.75,"price_out":14.0,"cheapest_provider":"azure","cheapest_model":"gpt-5.2-codex","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-luna-low","name":"GPT-5.6 Luna (low)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":5.14,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":2.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":80,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":1.549,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":41.404,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":82.323,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":43.79,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":66.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":46.065,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":12,"released_at":"2026-07-09","release_source":"epoch","price_in":0.2,"price_out":1.2,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.6-luna","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-luna-xhigh","name":"GPT-5.6 Luna (xhigh)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":47.64,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":20.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":89.07,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":56.858,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1457,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1260,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":48.54,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":50.463,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":46.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1518.87,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":5.158,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":12,"released_at":"2026-07-09","release_source":"epoch","price_in":0.2,"price_out":1.2,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.6-luna","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-oss-120b","name":"gpt-oss-120b","org":"OpenAI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":575.62,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":4.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":41.8,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":140.23,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":36.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":80.80808,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":41.983,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":80.8,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":26,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":22.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":18.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":12,"released_at":"2025-08-05","release_source":"curated","price_in":0.03,"price_out":0.17,"cheapest_provider":"wandb","cheapest_model":"openai/gpt-oss-120b","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-3-high","name":"grok-4.3 (high)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":18,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":90.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":42.807,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":14.634,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":88.826,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":38.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":93.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":11,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":48.264,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":33.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":12,"released_at":"2026-04-17","release_source":"epoch","price_in":1.25,"price_out":2.5,"cheapest_provider":"azure_ai","cheapest_model":"grok-4.3","price_observed_at":"2026-11-18T15:47Z"},{"id":"inkling-xhigh","name":"Inkling (xhigh)","org":"Thinking Machines","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":21,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":5.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":87.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":33.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":4.878,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":4.119,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":88.258,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":37.58,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":88.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":46.991,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":40.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":3.903,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":12,"released_at":"2026-07-15","release_source":"epoch","price_in":0.95,"price_out":4.05,"cheapest_provider":"deepinfra","cheapest_model":"thinkingmachines/Inkling","price_observed_at":"2026-11-18T15:47Z"},{"id":"mimo-v2-5-pro","name":"mimo-v2.5-pro","org":"Xiaomi Corp","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":899.8,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":84.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":66.7,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":99.6,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1468,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":29.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":68.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":22,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"fresh, decontaminated GitHub-issue tasks (resolved %)","label":"SWE-rebench","trust":2,"value":46.5,"score_max":100,"source_url":"https://swe-rebench.com/","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_rebench","source_label":"SWE-rebench leaderboard","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":50.231,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1474.92,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":12,"released_at":"2026-04-22","release_source":"openrouter","price_in":0.3045,"price_out":0.609,"cheapest_provider":"gmi","cheapest_model":"XiaomiMiMo/MiMo-V2.5-Pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark","name":"Muse Spark","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":11.331,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":153.16,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":89.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":40.56,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1444,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1489,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1294,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"tool use across 1,000 MCP tasks","label":"MCP Atlas","trust":4,"value":82.2,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/mcp_atlas","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"mcp_atlas","source_label":"Scale Labs MCP Atlas leaderboard","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":88.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":51.505,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1512.53,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":12,"released_at":"2026-04-08","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"muse-wonder-max","name":"Muse Wonder (max)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":66.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":28.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":50.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":31.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":82.105,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":70.732,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":54.167,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":54.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":45.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":51,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":99.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":60.301,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":12,"released_at":"2026-10-12","release_source":"epoch","price_in":4.0,"price_out":20.0,"cheapest_provider":"openrouter","cheapest_model":"meta/muse-wonder","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-5-plus","name":"Qwen3.5 Plus","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":608.92,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":19.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning from everyday-life context","label":"CL-bench Life","trust":2,"value":12.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench_life","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":80.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":84.848,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":36.36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":86.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":25.351,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":0.544,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1398.92,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":12,"released_at":"2026-02-16","release_source":"epoch","price_in":0.36,"price_out":2.21,"cheapest_provider":"empiriolabs","cheapest_model":"qwen3-5-plus","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-5-sonnet-oct-2024","name":"Claude 3.5 Sonnet (Oct 2024)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":32.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":67.84,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":0.91,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":55.303,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":4.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":4.08,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":20.523,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":8.472,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":41.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"work tasks in a simulated software company","label":"TheAgentCompany","trust":2,"value":24,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"agent_company","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":39.97,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2024-10-22","release_source":"epoch","price_in":3.0,"price_out":15.0,"cheapest_provider":"snowflake","cheapest_model":"claude-3-5-sonnet","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-1-high","name":"Claude Fable 5.1 (high)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":2143.2,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":59.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":88.75,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":30.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":49.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":96.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":29.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":65.09,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":73.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":58.681,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":92.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2026-09-01","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5-1","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-5-no-thinking","name":"Claude Opus 4.5 (no thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":43.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":80.682,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":26.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":292.995,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks (v1)","label":"OSWorld","trust":2,"value":66.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"os_world","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":48.056,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":76.653,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":62,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":63.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1468.18,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2025-11-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-5-unknown-thinking","name":"Claude Opus 4.5 (unknown thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":43,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":21.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":82,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":11.91,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":26.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":25.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":3.75,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":62,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":63.146,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":4967.057,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2025-11-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-pro-high","name":"DeepSeek v4 Pro (high)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1006.08,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning from everyday-life context","label":"CL-bench Life","trust":2,"value":13.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench_life","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":10,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":90.909,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":95.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"fresh, decontaminated GitHub-issue tasks (resolved %)","label":"SWE-rebench","trust":2,"value":40.2,"score_max":100,"source_url":"https://swe-rebench.com/","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_rebench","source_label":"SWE-rebench leaderboard","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":46.412,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":40,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1463.61,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":46.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2026-04-24","release_source":"epoch","price_in":0.528,"price_out":1.584,"cheapest_provider":"clf_ai_gateway","cheapest_model":"deepseek-v4-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v3-mar-2025","name":"DeepSeek-V3 (Mar 2025)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":64.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":135.28,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":50,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":67.614,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":15.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":23.117,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":37.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":39.005,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":27.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":36.08,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2025-03-24","release_source":"epoch","price_in":0.19,"price_out":0.38,"cheapest_provider":"inferenceapis","cheapest_model":"deepseek-ai/DeepSeek-V3","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-pro","name":"DeepSeek-V4-Pro","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":1000000,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":150.21,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":90.1,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":92.6,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":42.626,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":69.994,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":74.53767,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":62.35,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":78.13033,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":90.6755,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":82.69225,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":87.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":11,"released_at":"2026-04-24","release_source":"epoch","price_in":0.528,"price_out":1.584,"cheapest_provider":"clf_ai_gateway","cheapest_model":"deepseek-v4-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-5-flash-high","name":"gemini-3.5-flash-high","org":"Google","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":72.08,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1461,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1477,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1284,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":48.989666,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":78.1845,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":64.85733,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":75.6,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":84.584335,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":88.24375,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":82.00475,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true}],"n_cells":11,"released_at":"2026-05-19","release_source":"epoch","price_in":1.5,"price_out":9.0,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-3.5-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4o-nov-2024","name":"GPT-4o (Nov 2024)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":12.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":0.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":128.78,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":47.885,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":2.72,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":6.25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":30.992,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"work tasks in a simulated software company","label":"TheAgentCompany","trust":2,"value":8.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"agent_company","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":25.12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2024-11-20","release_source":"epoch","price_in":2.5,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-4o","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-medium","name":"GPT-5 (medium)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":7.49,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":32,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":48.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":97.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":85.354,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":137.319,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":15,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":87.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":71.488,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":49.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1419.52,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2025-08-07","release_source":"curated","price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-2-medium","name":"GPT-5.2 (medium)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1249.83,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":26.67,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":87.879,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":22,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":93.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":2.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":32.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":64.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1394,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":63.44,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2025-12-11","release_source":"curated","price_in":1.75,"price_out":14.0,"cheapest_provider":"azure","cheapest_model":"gpt-5.2","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4-nano-high","name":"GPT-5.4 nano (high)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1004.52,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":3.61,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":44.912,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":12.195,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":78.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":87.78,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":11.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":49.23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2026-03-17","release_source":"epoch","price_in":0.2,"price_out":1.25,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-nano","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-5","name":"GPT-5.5","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":1050000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":160.19,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1486,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1477,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1287,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":41.111,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":78.6285,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":76.987,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":65.7375,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":85.570335,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":69.7695,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":87.274,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true}],"n_cells":11,"released_at":"2026-04-23","release_source":"epoch","price_in":5.0,"price_out":30.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-luna-medium","name":"GPT-5.6 Luna (medium)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":7.36,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":4.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":22.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":83.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":11.283,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":22.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":44.25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"fresh, decontaminated GitHub-issue tasks (resolved %)","label":"SWE-rebench","trust":2,"value":43.6,"score_max":100,"source_url":"https://swe-rebench.com/","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_rebench","source_label":"SWE-rebench leaderboard","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":46.759,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":61.875,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2026-07-09","release_source":"epoch","price_in":0.2,"price_out":1.2,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.6-luna","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-sol-low","name":"GPT-5.6 Sol (low)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":42.5,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":31,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":14.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":24.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":93.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":45.354,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":89.899,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":55.89,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":95.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":56.366,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2026-07-09","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"replicate","cheapest_model":"openai/gpt-5.6-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-terra-low","name":"GPT-5.6 Terra (low)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":18.75,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":9.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":25.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":90.13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":24.053,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":87.374,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":49.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":19,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":88.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":49.884,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2026-07-09","release_source":"epoch","price_in":2.0,"price_out":12.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.6-terra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-astra-unknown-thinking","name":"GPT-6 Astra (unknown thinking)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":64.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":49.675,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":97.07,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":87.053,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":79.41,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":54.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":64.11,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":99,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":20.83,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":83.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":15514.702,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":null,"release_source":null,"price_in":10.0,"price_out":50.0,"cheapest_provider":"azure","cheapest_model":"gpt-6-astra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-2-astra-max","name":"GPT-6.2 Astra (max)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":95.833336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":78,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":92.381,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":92.632,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":89.167,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":96.843,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":67.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":95,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":78.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2026-11-17","release_source":"epoch","price_in":8.0,"price_out":40.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.2-astra","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-20","name":"grok-4-20","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1150.28,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":22.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning from everyday-life context","label":"CL-bench Life","trust":2,"value":11.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench_life","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":90.13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":152.3,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":38.66,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":57.303,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":4662.847,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1373.76,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":52.26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2026-02-17","release_source":"epoch","price_in":1.25,"price_out":2.5,"cheapest_provider":"openrouter","cheapest_model":"x-ai/grok-4.20","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-4-maverick","name":"Llama 4 Maverick","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":15.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":61.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":0.58,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":132.24,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":5.68,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":15.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":31.713,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":27.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":24.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2025-04-05","release_source":"curated","price_in":0.2,"price_out":0.6,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"meta/llama-4-maverick","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-3-3-70b-instruct","name":"Llama-3.3-70B-Instruct","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":59.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":59.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":33.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":47.443,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":17.52,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":5.139,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":26.042,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":19.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":14.44,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2024-12-06","release_source":"epoch","price_in":0.12,"price_out":0.3,"cheapest_provider":"hyperbolic","cheapest_model":"meta-llama/Llama-3.3-70B-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"minimax-m2-5","name":"MiniMax-M2.5","org":"MiniMax","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":618.17,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":4.86,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":11.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning from everyday-life context","label":"CL-bench Life","trust":2,"value":6.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench_life","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":85.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":80.1,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Multilingual","trust":4,"value":68.3,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multilingual","source_label":"SWE-bench Multilingual leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":75.8,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":42.697,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1384.13,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2026-02-12","release_source":"curated","price_in":0.295,"price_out":1.2,"cheapest_provider":"openrouter","cheapest_model":"minimax/minimax-m2.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"minimax-m2-7","name":"MiniMax-M2.7","org":"MiniMax","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":599.25,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing working exploits","label":"ExploitBench","trust":2,"value":13.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"exploitbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":89.8,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":81.8,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":50.116,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":45.056,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1397.52,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":36.95,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2026-03-18","release_source":"epoch","price_in":0.15,"price_out":0.6,"cheapest_provider":"empiriolabs","cheapest_model":"minimax-m2-7","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-medium-3-5","name":"Mistral Medium 3.5","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":763.98,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":75.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":141.57,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":26.06,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":40.162,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":26.875,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1264.92,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":43.71,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2026-04-28","release_source":"epoch","price_in":1.5,"price_out":7.5,"cheapest_provider":"azure_ai","cheapest_model":"mistral-medium-3-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen-3-6-max-preview","name":"Qwen 3.6 Max (Preview)","org":"Alibaba","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":87.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":149.88,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":87.374,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":42.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":19,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":91.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":76.653,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":63,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":52.004,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":4254.187,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1479.06,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-max-instruct","name":"Qwen3-Max-Instruct","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":370.45,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":14.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":82.13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":66.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":18.947,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":72.601,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":28.27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":73.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":48.749,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":71.565,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2025-09-24","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-5-397b-a17b","name":"Qwen3.5 397B-A17B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":621.92,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":24.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":87.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":147.95,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":31.228,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":88.4,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":37.89,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":87.8,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":18,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":88.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2026-02-13","release_source":"epoch","price_in":0.1644,"price_out":0.9864,"cheapest_provider":"aihubmix","cheapest_model":"qwen3.5-397b-a17b","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-5-flash","name":"Qwen3.5 Flash","org":"Alibaba","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":265.93,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":82.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":18.246,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":82.323,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":29.14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":20,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":84.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":20.32,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":462.685,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1238.17,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":11,"released_at":"2026-02-25","release_source":"epoch","price_in":0.065,"price_out":0.26,"cheapest_provider":"openrouter","cheapest_model":"qwen/qwen3.5-flash-02-23","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-6-no-thinking","name":"Claude Opus 4.6 (no thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":996.5,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":6.84,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":33.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":19,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":15,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":78.719,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":67.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":69.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1537.13,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":65.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":10,"released_at":"2026-02-05","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-5-low","name":"Claude Opus 5 (low)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":23.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":40.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":93.07,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":58.129,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":87.879,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":62.58,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks","label":"OSWorld 2","trust":2,"value":22.32,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"osworld_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":93.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":48.032,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":10,"released_at":"2026-07-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"aihubmix","cheapest_model":"claude-opus-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-5-xhigh","name":"Claude Sonnet 5 (xhigh)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":34,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":15.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":32,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":89.51,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":49.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":90.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":48.81,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":94.722,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":54.051,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":32.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":10,"released_at":"2026-06-30","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"aihubmix","cheapest_model":"claude-sonnet-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-1-20250805","name":"claude-opus-4-1-20250805","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":42,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":48.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":7.937,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":73.232,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":100.472,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":40,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":73.347,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":60,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":38,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1386,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":10,"released_at":"2025-08-05","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-v3","name":"DeepSeek-V3","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":131.71,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":58.20707,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":89.3,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":18.473,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":64.4,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":15.833,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":4,"value":30.67,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":35.764,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":18.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":10,"released_at":"2024-12-26","release_source":"curated","price_in":0.19,"price_out":0.38,"cheapest_provider":"inferenceapis","cheapest_model":"deepseek-ai/DeepSeek-V3","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-flash","name":"DeepSeek-V4-Flash","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":1000000,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":146.84,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":88.1,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":37.626,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":69.228,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":68.023,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":63.1375,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":70.124,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":79.6475,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":70.58175,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":86.4,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":10,"released_at":"2026-04-24","release_source":"epoch","price_in":0.09,"price_out":0.18,"cheapest_provider":"deepinfra","cheapest_model":"deepseek-ai/DeepSeek-V4-Flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-pro-preview-jun-2025","name":"Gemini 2.5 Pro Preview (Jun 2025)","org":"Google DeepMind","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":42.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":5.57,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":91.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":84.848,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":3.92,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":21.64,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":38.734,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":0.83,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":62.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"work tasks in a simulated software company","label":"TheAgentCompany","trust":2,"value":30.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"agent_company","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":10,"released_at":"2025-05-06","release_source":"epoch","price_in":1.25,"price_out":10.0,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemma-4-26b-a4b","name":"Gemma 4 26B A4B","org":"Google DeepMind","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":927.17,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":74.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":141.58,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":73.232,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":29.68,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":82.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":40.046,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1361.4,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":35.17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":10,"released_at":"2026-04-02","release_source":"epoch","price_in":0.05,"price_out":0.29,"cheapest_provider":"empiriolabs","cheapest_model":"gemma-4-26b-a4b","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-minimal","name":"GPT-5 (minimal)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":807.65,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":32.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":11,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":48.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":18.246,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":71.717,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":46.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":10,"released_at":"2025-08-07","release_source":"curated","price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-5-unknown-thinking","name":"GPT-5.5 (unknown thinking)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":55.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":36.226,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing working exploits","label":"ExploitBench","trust":2,"value":47.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"exploitbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":42.96,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":53.22,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":6.25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":69,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":84.719,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":7523.844,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1457.76,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":10,"released_at":null,"release_source":null,"price_in":5.0,"price_out":30.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-sol-high","name":"GPT-5.6 Sol (high)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":85.42,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":25.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":35.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":95.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":69.401,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":37.12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":56.94,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":20,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":57.755,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":88.76,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":10,"released_at":"2026-07-09","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"replicate","cheapest_model":"openai/gpt-5.6-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-0709","name":"grok-4-0709","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":43.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":43,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":47.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":94.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":110.075,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":84,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":60.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":27.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":45.73,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":10,"released_at":"2025-07-09","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"grok-build-0-1","name":"grok-build-0.1","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":9.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":2.381,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":45.808,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":65.3855,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":70.794,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":65.22075,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":72.46,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":78.42875,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":76.37025,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":50.231,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":10,"released_at":"2026-05-29","release_source":"epoch","price_in":1.0,"price_out":2.0,"cheapest_provider":"aihubmix","cheapest_model":"grok-build-0.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"o1-preview","name":"o1-preview","org":"OpenAI","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":79.7,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"Aider official benchmark","label":"Aider Refactoring","trust":4,"value":75.3,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/refactor.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_refactor","source_label":"Aider Aider Refactoring leaderboard","higher_is_better":true},{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":10,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":133.54,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":50.316,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":20.327,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":31.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":64.6,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":41.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":47.56,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":10,"released_at":null,"release_source":null,"price_in":15.0,"price_out":60.0,"cheapest_provider":"azure","cheapest_model":"o1","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen-3-6-flash-2026-04-16","name":"Qwen 3.6 Flash (2026-04-16)","org":"Alibaba","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":326.4,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":77.07,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":22.456,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":83.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":30.98,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":18,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":84.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":35.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":15.916,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":10,"released_at":"2026-04-16","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-235b-a22b-thinking-2507","name":"Qwen3-235B-A22B-Thinking-2507","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":80.27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":75,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":80.051,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":29.31,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":86.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":41.435,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":40.44,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":41.04,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":10,"released_at":null,"release_source":null,"price_in":0.3,"price_out":2.9,"cheapest_provider":"deepinfra","cheapest_model":"Qwen/Qwen3-235B-A22B-Thinking-2507","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-8-max-0902-xhigh","name":"Qwen3.8 Max (0902) (xhigh)","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1679.12,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":37,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":65.614,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":34.146,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":17.81,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":20,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":92.298,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":47.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":47.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":10,"released_at":null,"release_source":null,"price_in":1.65,"price_out":4.951,"cheapest_provider":"deepinfra","cheapest_model":"Qwen/Qwen3.8-Max","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-haiku-4-5-unknown-thinking","name":"Claude Haiku 4.5 (unknown thinking)","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":73.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing working exploits","label":"ExploitBench","trust":2,"value":13.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"exploitbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":30.94,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":42.245,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":35.506,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":458.886,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1329.31,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2025-10-15","release_source":"curated","price_in":1.0,"price_out":5.0,"cheapest_provider":"anthropic","cheapest_model":"claude-haiku-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-6-high","name":"Claude Opus 4.6 (high)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"learning from everyday-life context","label":"CL-bench Life","trust":2,"value":17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench_life","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":55.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":41.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1546,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1507,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1505,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1299,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1546.62,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":77.95,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2026-02-05","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-7-unknown","name":"Claude Opus 4.7 (unknown)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":24.512,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":38.54,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":43.811,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":36.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":61.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":80.225,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":10936.763,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1566.85,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":76.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2026-04-16","release_source":"epoch","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-7","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-5-5-high","name":"Claude Opus 5.5 (High)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":2147.38,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":93.333336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":30.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":56,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":96.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1504,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":65.91,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":78.89,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":60.417,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2026-09-22","release_source":"curated","price_in":4.0,"price_out":20.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-5-5-high","name":"Claude Sonnet 5.5 (high)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1819.12,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":67.361115,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":24.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":47.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":94.13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1716,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":65.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":53.704,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1698.82,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2026-09-28","release_source":"curated","price_in":2.0,"price_out":10.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-20250514","name":"claude-sonnet-4-20250514","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":61.3,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":35,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":46.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":66.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":4.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks (v1)","label":"OSWorld","trust":2,"value":43.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"os_world","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":28.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"work tasks in a simulated software company","label":"TheAgentCompany","trust":2,"value":33.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"agent_company","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":43.88,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2025-05-22","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-r1-may-2025","name":"DeepSeek-R1 (May 2025)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":804.12,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":35.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":140.75,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":75,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":76.326,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":31.168,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":66.389,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":40.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":41.63,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2025-05-28","release_source":"epoch","price_in":0.4,"price_out":0.4,"cheapest_provider":"hyperbolic","cheapest_model":"deepseek-ai/DeepSeek-R1","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-8-flash-medium","name":"Gemini 3.8 Flash (medium)","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":82.916664,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":12.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":37.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":94.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":71.018,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":41.19,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":23.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":52.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":55.093,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2026-09-02","release_source":"epoch","price_in":0.75,"price_out":3.75,"cheapest_provider":"gemini","cheapest_model":"gemini-3.8-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-8-flash-unknown","name":"Gemini 3.8 Flash (unknown)","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":64.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":38.612,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":95.72,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":44.52,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":51.55,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":48,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":5.83,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":82.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":5093.793,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2026-09-02","release_source":"epoch","price_in":0.75,"price_out":3.75,"cheapest_provider":"gemini","cheapest_model":"gemini-3.8-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-1-5-pro-002","name":"gemini-1.5-pro-002","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":65.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":21,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":58.98,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":57.229,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":4.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":23.056,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":27.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"work tasks in a simulated software company","label":"TheAgentCompany","trust":2,"value":3.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"agent_company","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":22.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2024-09-24","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-5-flash","name":"gemini-2.5-flash","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":1048576,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":661.88,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":33.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":1.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":76.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":27.49,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":28.73,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":41.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":17.079,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":548.836,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2025-04-17","release_source":"curated","price_in":0.3,"price_out":2.5,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-1-unknown-thinking","name":"GPT-5.1 (unknown thinking)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":21.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":4.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":11.23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":13.73,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":23.68,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":43.287,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":47.584,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":1473.432,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1340.9,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":null,"release_source":null,"price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"global/gpt-5.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4-mini-high","name":"GPT-5.4 mini (high)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1188.58,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":13.19,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":83.586,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1251,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":87.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":29.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1397.18,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":60.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2026-03-17","release_source":"epoch","price_in":0.75,"price_out":4.5,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-5-low","name":"GPT-5.5 (low)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":33.33,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":30,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":48.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":26.991,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":90.657,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":28,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":84.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":51.62,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2026-04-23","release_source":"epoch","price_in":5.0,"price_out":30.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-5-instant","name":"GPT-5.5 Instant","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":26.316,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":2.439,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":82.513,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1403,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1473,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1276,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":68.056,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":52.546,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2026-05-05","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-5-6-luna-none","name":"GPT-5.6 Luna (none)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":69.25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":39.649,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":63.636,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":41.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":20,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":40,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":40.394,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2026-07-09","release_source":"epoch","price_in":0.2,"price_out":1.2,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.6-luna","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-terra-medium","name":"GPT-5.6 Terra (medium)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":37.5,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":17.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":27.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":90.13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":35.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":24.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":49.64,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":10,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":50.463,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2026-07-09","release_source":"epoch","price_in":2.0,"price_out":12.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.6-terra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-astra-high","name":"GPT-6 Astra (high)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":92.083336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":28.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":96.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":73.23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":97.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":63.84,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":46.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":55.44,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":92.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2026-09-03","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"azure","cheapest_model":"gpt-6-astra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-oss-120b-high","name":"gpt-oss-120b (high)","org":"OpenAI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":1.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":76.27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":44.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":75.758,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":22.14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":88.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":34.028,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":48.17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2025-08-05","release_source":"curated","price_in":0.03,"price_out":0.17,"cheapest_provider":"wandb","cheapest_model":"openai/gpt-oss-120b","price_observed_at":"2026-11-18T15:47Z"},{"id":"inkling","name":"Inkling","org":"Thinking Machines","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":946,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":33.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":36.52778,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":148.43,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":87.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":50,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1409.99,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2026-07-15","release_source":"epoch","price_in":0.95,"price_out":4.05,"cheapest_provider":"deepinfra","cheapest_model":"thinkingmachines/Inkling","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k2-thinking","name":"Kimi K2 Thinking","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":597.5,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":11.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":145.48,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":84.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":54.194,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":63.4,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":39.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":35.73,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":42.79,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":null,"release_source":null,"price_in":0.55,"price_out":2.2,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"moonshotai/kimi-k2","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k3-unknown","name":"Kimi K3 (unknown)","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":47.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":29.501,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":44.17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":48.347,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":31.48,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":58.681,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":95,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":5165.035,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2026-07-16","release_source":"epoch","price_in":0.192,"price_out":16.0,"cheapest_provider":"openrouter","cheapest_model":"moonshotai/kimi-k3","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-3-1-405b-instruct","name":"Llama-3.1-405B-Instruct","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":66.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":7.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":61.38,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":50.915,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":9.722,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"work tasks in a simulated software company","label":"TheAgentCompany","trust":2,"value":7.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"agent_company","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":21.38,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2024-07-23","release_source":"epoch","price_in":3.5,"price_out":3.5,"cheapest_provider":"together_ai","cheapest_model":"meta-llama/Llama-3.1-405B-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-3-1-8b-instruct","name":"Llama-3.1-8B-Instruct","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":37.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":15.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":50.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":26.957,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":5.37,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":1.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":13.194,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":1.73,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2024-07-23","release_source":"epoch","price_in":0.02,"price_out":0.04,"cheapest_provider":"openrouter","cheapest_model":"meta-llama/llama-3.1-8b-instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark-open-xhigh","name":"Muse Spark Open (xhigh)","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1590.12,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":57.5,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":14.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":69.825,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":31.707,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":88.573,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":98.056,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":58.449,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":56.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2026-10-15","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"aiand","cheapest_model":"meta/muse-spark-open","price_observed_at":"2026-11-18T15:47Z"},{"id":"o1-high","name":"o1 (high)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":10,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":74.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":14.737,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":76.768,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":22.27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":73.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":40.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":41.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":46.08,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2024-12-05","release_source":"curated","price_in":15.0,"price_out":60.0,"cheapest_provider":"azure","cheapest_model":"o1","price_observed_at":"2026-11-18T15:47Z"},{"id":"o4-mini-medium","name":"o4-mini (medium)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":2.36,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":22,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":6.81,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":77.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":28.772,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":77.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":14.28,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":76.509,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":73.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2025-04-16","release_source":"curated","price_in":1.0,"price_out":4.0,"cheapest_provider":"replicate","cheapest_model":"openai/o4-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-32b","name":"Qwen3-32B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":40,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":67.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":136.84,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":74.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":65.72,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":17.28,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":66.944,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":35.417,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":9,"released_at":"2025-04-29","release_source":"curated","price_in":0.08,"price_out":0.23,"cheapest_provider":"ovhcloud","cheapest_model":"Qwen3-32B","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-low","name":"Claude Fable 5 (low)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":76.80556,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":31,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":91.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":59.584,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":78.788,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":59.59,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":97.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":5018.52,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-06-09","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-1-unknown","name":"Claude Fable 5.1 (unknown)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":68.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":41.866,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":97.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":88.24,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":65.46,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":17.92,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":86.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":5421.557,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-09-01","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5-1","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-1-xhigh","name":"Claude Fable 5.1 (xhigh)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":90,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":31.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":51.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":97.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":46.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":65.06,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":60.88,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":25.97,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-09-01","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5-1","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-2-unknown","name":"Claude Fable 5.2 (unknown)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":80.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":56.616,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":99.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":84.801,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":70.08,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":41.25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":94.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":9628.48,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-11-09","release_source":"epoch","price_in":8.0,"price_out":40.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5-2","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-haiku-5-5-high","name":"Claude Haiku 5.5 (high)","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1587,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":30.694445,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":18.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":42.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":86.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":45.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":97.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":48.727,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-10-07","release_source":"epoch","price_in":0.1,"price_out":0.5,"cheapest_provider":"anthropic","cheapest_model":"claude-haiku-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-haiku-5-5-xhigh","name":"Claude Haiku 5.5 (xhigh)","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":61.25,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":22.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":44.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":87.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":47.99,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":98.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":51.736,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":11.695,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-10-07","release_source":"epoch","price_in":0.1,"price_out":0.5,"cheapest_provider":"anthropic","cheapest_model":"claude-haiku-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-7-high","name":"Claude Opus 4.7 (high)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":44.12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1557,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1495,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1501,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1298,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"reimplementing programs from behaviour","label":"MirrorCode","trust":2,"value":31.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mirrorcode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1555.22,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":76.44,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-04-16","release_source":"epoch","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-7","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-5-xhigh","name":"Claude Opus 5 (xhigh)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":27.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":46.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":97.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":73.154,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":64.45,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks","label":"OSWorld 2","trust":2,"value":30.21,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"osworld_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":54.977,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":19.011,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-07-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"aihubmix","cheapest_model":"claude-opus-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-5-5-medium","name":"Claude Opus 5.5 (medium)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":52.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":87.5,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":27.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":52.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":96.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":54.64,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":65.66,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":59.259,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-09-22","release_source":"curated","price_in":4.0,"price_out":20.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-unknown-thinking","name":"Claude Sonnet 4 (unknown thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":77.07,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":3.12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":4.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":7.76,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":29,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":40.046,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":45.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2025-05-22","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"gmi","cheapest_model":"anthropic/claude-sonnet-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-6-high","name":"Claude Sonnet 4.6 (high)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":43,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":60.42,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":19,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":54.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":29.933,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":83.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":75.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":35.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-02-17","release_source":"epoch","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-haiku-4-5-20251001","name":"claude-haiku-4-5-20251001","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":200000,"open_weight":false,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":31.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":60.48,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1420,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":35.833,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":13.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":29.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1286,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":45.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2025-10-15","release_source":"curated","price_in":1.0,"price_out":5.0,"cheapest_provider":"anthropic","cheapest_model":"claude-haiku-4-5-20251001","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-pro-unknown-thinking","name":"DeepSeek v4 Pro (unknown thinking)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":521.67,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":17.64,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":73.232,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":46.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":3284.52,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1463.63,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":null,"release_source":null,"price_in":0.528,"price_out":1.584,"cheapest_provider":"clf_ai_gateway","cheapest_model":"deepseek-v4-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v3-2-exp-thinking","name":"DeepSeek-V3.2-Exp (thinking)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":13.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning from everyday-life context","label":"CL-bench Life","trust":2,"value":9.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench_life","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":2.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":87.73,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":29.11,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":38.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1367,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":39.48,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":null,"release_source":null,"price_in":0.27,"price_out":0.41,"cheapest_provider":"gmi","cheapest_model":"deepseek-ai/DeepSeek-V3.2-Exp","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-1-pro-preview-high","name":"gemini-3.1-pro-preview-high","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":45.404,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":76.4545,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":78.541336,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":79.1,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":85.376,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":91.045,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":84.00475,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"tool use across 1,000 MCP tasks","label":"MCP Atlas","trust":4,"value":78.2,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/mcp_atlas","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"mcp_atlas","source_label":"Scale Labs MCP Atlas leaderboard","higher_is_better":true}],"n_cells":8,"released_at":null,"release_source":null,"price_in":2.0,"price_out":12.0,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3.1-pro-preview","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemma-3-27b-it","name":"gemma-3-27b-it","org":"Google DeepMind","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":4.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":52.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":33.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":47.727,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":12.28,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":22.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":23.264,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2025-03-12","release_source":"curated","price_in":0.08,"price_out":0.16,"cheapest_provider":"deepinfra","cheapest_model":"google/gemma-3-27b-it","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-3-5-turbo-jan-2024","name":"GPT-3.5 Turbo (Jan 2024)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":48.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":112.77,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":27.178,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":9.65,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":2.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":3.48,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2024-01-25","release_source":"epoch","price_in":0.5,"price_out":1.5,"cheapest_provider":"azure","cheapest_model":"gpt-3.5-turbo","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4-turbo-2024-04-09","name":"gpt-4-turbo-2024-04-09","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":128000,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":63.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"Aider official benchmark","label":"Aider Refactoring","trust":4,"value":34.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/refactor.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_refactor","source_label":"Aider Aider Refactoring leaderboard","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":0.702,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":46.591,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":6.567,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":6.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":25.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":18.01,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2024-04-09","release_source":"name","price_in":10.0,"price_out":30.0,"cheapest_provider":"openai","cheapest_model":"gpt-4-turbo-2024-04-09","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4-5-preview-feb-2025","name":"GPT-4.5 Preview (Feb 2025)","org":"OpenAI","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":17.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":3.18,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":63.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":68.687,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":5.44,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":37.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":34.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":39.37,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2025-02-27","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-5-2-low","name":"GPT-5.2 (low)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":9.72,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":41.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":82.702,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":10,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":78.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":32.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":49.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2025-12-11","release_source":"curated","price_in":1.75,"price_out":14.0,"cheapest_provider":"azure","cheapest_model":"gpt-5.2","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-2-unknown-thinking","name":"GPT-5.2 (unknown thinking)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":18.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":10.39,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":27.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":2.08,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":45.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":64.944,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":3591.326,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1416.64,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":null,"release_source":null,"price_in":1.75,"price_out":14.0,"cheapest_provider":"azure","cheapest_model":"gpt-5.2","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-5-no-thinking","name":"GPT-5.5 (no thinking)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1127.58,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":1.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":77.273,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":18,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":57.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":47.338,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":67.15,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":null,"release_source":null,"price_in":5.0,"price_out":30.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-luna-high","name":"GPT-5.6 Luna (high)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":29.31,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":16.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":29.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":86.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":44.248,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":47.38,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":51.62,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":60.86,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-07-09","release_source":"epoch","price_in":0.2,"price_out":1.2,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.6-luna","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-sol-medium","name":"GPT-5.6 Sol (medium)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":67.08,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":22.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":31.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":95.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":61.062,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":56.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"fresh, decontaminated GitHub-issue tasks (resolved %)","label":"SWE-rebench","trust":2,"value":62.3,"score_max":100,"source_url":"https://swe-rebench.com/","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_rebench","source_label":"SWE-rebench leaderboard","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":57.407,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-07-09","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"replicate","cheapest_model":"openai/gpt-5.6-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-sol-none","name":"GPT-5.6 Sol (none)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":5.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":83.73,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":82.828,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":52.75,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":68.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":47.106,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-07-09","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"replicate","cheapest_model":"openai/gpt-5.6-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-terra-high","name":"GPT-5.6 Terra (high)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":67.08,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":22.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":30.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":91.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":53.761,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":51.37,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":52.431,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":78.27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-07-09","release_source":"epoch","price_in":2.0,"price_out":12.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.6-terra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-terra-none","name":"GPT-5.6 Terra (none)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":78.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":77.273,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":46.51,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":53.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":45.139,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-07-09","release_source":"epoch","price_in":2.0,"price_out":12.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.6-terra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-astra-xhigh","name":"GPT-6 Astra (xhigh)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":93.333336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":31.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":97.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":74.115,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":97.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":64.37,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":55.671,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":42.22,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-09-03","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"azure","cheapest_model":"gpt-6-astra","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-5-unknown-thinking","name":"Grok 4.5 (unknown thinking)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":56.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":27.332,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":42.44,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":65.413,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":70,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":3887.426,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1555.1,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":46.43,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":null,"release_source":null,"price_in":2.0,"price_out":6.0,"cheapest_provider":"github_copilot","cheapest_model":"grok-4.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k3-1-pro-together-max","name":"Kimi K3.1 Pro (Together, max)","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":45,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":78.246,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":65.854,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":55.833,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":94.192,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":42,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":100,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":62.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-10-23","release_source":"epoch","price_in":0.2403,"price_out":8.4023,"cheapest_provider":"openrouter","cheapest_model":"moonshotai/kimi-k3.1-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-3-1-70b-instruct","name":"Llama-3.1-70B-Instruct","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":58.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":27.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":60,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":44.192,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":14.85,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":3.611,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"work tasks in a simulated software company","label":"TheAgentCompany","trust":2,"value":6.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"agent_company","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":8.97,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2024-07-23","release_source":"epoch","price_in":0.72,"price_out":0.72,"cheapest_provider":"openrouter","cheapest_model":"meta-llama/llama-3.1-70b-instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"mimo-v2-6-pro","name":"mimo-v2.6-pro","org":"Xiaomi Corp","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1158.1,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":54.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":26.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1630,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1480,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":70,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":60.88,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1618.41,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-09-21","release_source":"epoch","price_in":0.348,"price_out":0.696,"cheapest_provider":"gmi","cheapest_model":"XiaomiMiMo/MiMo-V2.6-Pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-large-4-high","name":"Mistral Large 4 (high)","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":29,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":10,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":18.095,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":30.833,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":90.152,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":20,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":99.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":22.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-10-06","release_source":"epoch","price_in":0.7,"price_out":2.1,"cheapest_provider":"azure_ai","cheapest_model":"mistral-large-4-latest","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen2-5-72b-instruct","name":"qwen2.5-72b-instruct","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":32768,"open_weight":true,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":16.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":62.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":49.148,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":13.42,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":5.166,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":8.056,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"work tasks in a simulated software company","label":"TheAgentCompany","trust":2,"value":5.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"agent_company","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":15.97,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2024-09-19","release_source":"epoch","price_in":0.12,"price_out":0.3,"cheapest_provider":"hyperbolic","cheapest_model":"Qwen/Qwen2.5-72B-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-14b","name":"qwen3-14b","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":64,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":136.08,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":62.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":63.763,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":18.19,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":66.389,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":31.597,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2025-04-28","release_source":"epoch","price_in":0.08,"price_out":0.24,"cheapest_provider":"nebius","cheapest_model":"Qwen/Qwen3-14B","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-235b-a22b","name":"qwen3-235b-a22b","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":75.73,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":138.43,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":67.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":70.707,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":68.18,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":31,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":37.28,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2025-04-29","release_source":"curated","price_in":0.18,"price_out":0.54,"cheapest_provider":"deepinfra","cheapest_model":"Qwen/Qwen3-235B-A22B","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-8b","name":"qwen3-8b","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":59.73,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":132.08,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":62.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":56.755,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":8.82,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":56.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":22.569,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2025-04-28","release_source":"epoch","price_in":0.04,"price_out":0.14,"cheapest_provider":"llamagate","cheapest_model":"qwen3-8b","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-5-27b","name":"Qwen3.5 27B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":349.45,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":82.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":85.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":33.98,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":86.1,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":201.976,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1357.38,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":39.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-02-24","release_source":"epoch","price_in":0.086,"price_out":0.688,"cheapest_provider":"empiriolabs","cheapest_model":"qwen3-5-27b","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-5-9b","name":"Qwen3.5 9B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":71.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":139.48,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":78.977,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":24.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":61.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":29.514,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":9.213,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-02-24","release_source":"epoch","price_in":0.09,"price_out":0.13,"cheapest_provider":"empiriolabs","cheapest_model":"qwen3-5-9b","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-8-27b","name":"Qwen3.8 27B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":47.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":88,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":89.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1591,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-07","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-06)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":41.36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":60.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1593.29,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":8,"released_at":"2026-08-14","release_source":"epoch","price_in":0.06,"price_out":2.2,"cheapest_provider":"openrouter","cheapest_model":"qwen/qwen3.8-27b","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-5-haiku-oct-2024","name":"Claude 3.5 Haiku (Oct 2024)","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":19.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":56.66,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":38.131,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":4.306,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":27.431,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":30.73,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2024-10-22","release_source":"epoch","price_in":0.8,"price_out":4.0,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"anthropic/claude-3.5-haiku","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-5-sonnet-jun-2024","name":"Claude 3.5 Sonnet (Jun 2024)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":17.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":67.77,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":54.04,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":11.395,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":6.528,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":27.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":30.97,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2024-06-20","release_source":"epoch","price_in":3.0,"price_out":15.0,"cheapest_provider":"snowflake","cheapest_model":"claude-3-5-sonnet","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-unknown","name":"Claude Fable 5 (unknown)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":38.612,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":53.48,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":74.472,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":30,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":78.43,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":15.83,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1626.26,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-06-09","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-1-medium","name":"Claude Fable 5.1 (medium)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":86.25,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":29.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":46.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":97.07,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":50.91,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":65.32,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":56.366,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-09-01","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5-1","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-2-high","name":"Claude Fable 5.2 (High)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":2509.8,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":90.416664,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":32.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":56.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":98.13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":69.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":59.491,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-11-09","release_source":"epoch","price_in":8.0,"price_out":40.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5-2","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-2-xhigh","name":"Claude Fable 5.2 (XHigh)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":93.75,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":35.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":56.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":98.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":70.61,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":65.625,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":43.089,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-11-09","release_source":"epoch","price_in":8.0,"price_out":40.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5-2","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-haiku-5-5-low","name":"Claude Haiku 5.5 (low)","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":2.7777777,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":24,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":9.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":30.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":81.07,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":44.83,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":49.19,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-10-07","release_source":"epoch","price_in":0.1,"price_out":0.5,"cheapest_provider":"anthropic","cheapest_model":"claude-haiku-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-1-unknown-thinking","name":"Claude Opus 4.1 (unknown thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":80,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":7.179,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":11.52,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":37.08,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":60,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":37.978,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1388.97,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2025-08-05","release_source":"epoch","price_in":15.0,"price_out":75.0,"cheapest_provider":"openrouter","cheapest_model":"anthropic/claude-opus-4.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-5-32k-thinking","name":"Claude Opus 4.5 (32k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":34.386,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":4.878,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":86.048,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":86.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":45.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1494.89,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2025-11-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-5-medium","name":"Claude Opus 5 (medium)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":26.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":43.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":94.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":68.904,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":62.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks","label":"OSWorld 2","trust":2,"value":25.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"osworld_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":50.694,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-07-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"aihubmix","cheapest_model":"claude-opus-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-5-5-xhigh","name":"Claude Opus 5.5 (XHigh)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":92.5,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":31.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":56,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":97.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":66.18,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":65.046,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":31.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-09-22","release_source":"curated","price_in":4.0,"price_out":20.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-5-32k-thinking","name":"Claude Sonnet 4.5 (32k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":796.15,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":10,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":23.86,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":2.439,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":81.726,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":77.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1391,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2025-09-29","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-5-medium","name":"Claude Sonnet 5 (medium)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":8.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":28,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":79.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":39.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":49.71,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":51.62,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":60,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-06-30","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"aihubmix","cheapest_model":"claude-sonnet-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-5-unknown-thinking","name":"Claude Sonnet 5 (unknown thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":54.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":24.946,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":42.73,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":65.263,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":37.25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":60.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":6377.702,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-06-30","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"aihubmix","cheapest_model":"claude-sonnet-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-5-5-medium","name":"Claude Sonnet 5.5 (medium)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":44.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":32.77778,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":16.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":39.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":93.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":65.45,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":52.894,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-09-28","release_source":"curated","price_in":2.0,"price_out":10.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-xhigh-effort","name":"claude-fable-5-xhigh-effort","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":50.656334,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":82.5325,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":78.693,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":71.99575,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":89.457664,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":95.693,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":87.7115,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true}],"n_cells":7,"released_at":"2026-06-09","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-5-20251101-thinking-64k-high-effort","name":"claude-opus-4-5-20251101-thinking-64k-high-effort","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":39.697,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":79.654,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":74.441666,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":62.546,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":81.261,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":90.389,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":80.0865,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true}],"n_cells":7,"released_at":"2025-11-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-5-20251101","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-6-thinking-auto-high-effort","name":"claude-opus-4-6-thinking-auto-high-effort","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":48.989666,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":78.1845,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":69.893,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":63.3125,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":83.26967,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":89.317,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":88.673,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true}],"n_cells":7,"released_at":"2026-02-05","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-7-xhigh-effort","name":"claude-opus-4-7-xhigh-effort","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":50.656334,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":82.088,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":78.263664,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":66.7375,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":77.914,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":92.85375,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":87.19225,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true}],"n_cells":7,"released_at":"2026-04-16","release_source":"epoch","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-7","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-8-xhigh-effort","name":"claude-opus-4-8-xhigh-effort","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":56.060665,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":79.2715,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":78.344,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":72.371,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":81.417336,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":95.3165,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":89.7115,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true}],"n_cells":7,"released_at":"2026-05-28","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-8","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-6-thinking-auto-medium-effort","name":"claude-sonnet-4-6-thinking-auto-medium-effort","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":42.626,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":79.2715,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":77.946,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":63.22075,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":76.10267,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":86.994,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":84.76925,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true}],"n_cells":7,"released_at":"2026-02-17","release_source":"epoch","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-5-xhigh-effort","name":"claude-sonnet-5-xhigh-effort","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":51.060665,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":80.68,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":71.74067,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":63.8585,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":74.97034,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":92.94225,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":88.69225,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true}],"n_cells":7,"released_at":"2026-06-30","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"aihubmix","cheapest_model":"claude-sonnet-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v3-2-thinking-novita","name":"DeepSeek-V3.2 (Thinking; Novita)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":21.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":12.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning from everyday-life context","label":"CL-bench Life","trust":2,"value":7.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench_life","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":85.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":28.82,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":39.551,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1324.67,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":null,"release_source":null,"price_in":0.26,"price_out":0.38,"cheapest_provider":"deepinfra","cheapest_model":"deepseek-ai/DeepSeek-V3.2","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-0-flash-feb-2025","name":"Gemini 2.0 Flash (Feb 2025)","org":"Google DeepMind,Google","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":63.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":134.02,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":61.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":64.141,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":31.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"work tasks in a simulated software company","label":"TheAgentCompany","trust":2,"value":11.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"agent_company","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":25.77,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2025-02-05","release_source":"epoch","price_in":0.15,"price_out":0.6,"cheapest_provider":"vertex_ai","cheapest_model":"gemini-2.0-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-pro","name":"Gemini 3 Pro","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":31.11,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":153.04,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1434,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1485,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1289,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Multilingual","trust":4,"value":68.7,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multilingual","source_label":"SWE-bench Multilingual leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":69.6,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":7,"released_at":"2025-11-18","release_source":"curated","price_in":2.0,"price_out":12.0,"cheapest_provider":"replicate","cheapest_model":"google/gemini-3-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-7-flash-medium","name":"Gemini 3.7 Flash (medium)","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":63.75,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":9.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":94.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":65.487,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":21.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":49.22,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":59.838,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-08-13","release_source":"epoch","price_in":0.75,"price_out":3.75,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3.7-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-4-argon-medium","name":"Gemini 4 Argon (medium)","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":87.083336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":23.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":52.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":97.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":47.55,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":62.85,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":58.912,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-09-30","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"gemini","cheapest_model":"gemini-4-argon","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-4-argon-unknown","name":"Gemini 4 Argon (unknown)","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":82.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":54.447,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":78.682,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":99,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":17.08,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":91.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":13718.158,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-09-30","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"gemini","cheapest_model":"gemini-4-argon","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4-jun-2023","name":"GPT-4 (Jun 2023)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":62.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":119.82,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":30.65,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":17.06,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":1.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":12.36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2023-06-13","release_source":"epoch","price_in":30.0,"price_out":60.0,"cheapest_provider":"openai","cheapest_model":"gpt-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4o-aug-2024","name":"GPT-4o (Aug 2024)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":127.12,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":0.351,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":49.211,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":6.389,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":17.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2024-08-06","release_source":"epoch","price_in":2.5,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-4o","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4o-may-2024","name":"GPT-4o (May 2024)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":32.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":64.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":128.13,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":48.895,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":16.59,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":9.17,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":6.25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2024-05-13","release_source":"epoch","price_in":2.5,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-4o","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-mini-medium","name":"GPT-5 mini (medium)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":4.03,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":69.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":71.654,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":78.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":64.669,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":31.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2025-08-07","release_source":"curated","price_in":0.25,"price_out":2.0,"cheapest_provider":"azure","cheapest_model":"gpt-5-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-nano-low","name":"GPT-5 nano (low)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":15,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":5.965,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":57.576,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":46.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":25.89,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2025-08-07","release_source":"curated","price_in":0.05,"price_out":0.4,"cheapest_provider":"azure","cheapest_model":"gpt-5-nano","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-2-2025-12-11-high","name":"gpt-5.2-2025-12-11-high","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":50.252666,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":76.0715,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":78.16333,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":61.77075,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":79.809,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":93.166,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":83.2115,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true}],"n_cells":7,"released_at":"2025-12-11","release_source":"name","price_in":1.75,"price_out":14.0,"cheapest_provider":"azure","cheapest_model":"gpt-5.2-2025-12-11","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4-medium","name":"GPT-5.4 (medium)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1520.72,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":55.42,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":37,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":88.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":28,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":95.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1442.65,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-03-05","release_source":"epoch","price_in":2.5,"price_out":15.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-high","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4-unknown-thinking","name":"GPT-5.4 (unknown thinking)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":52.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":27.115,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning from everyday-life context","label":"CL-bench Life","trust":2,"value":13.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench_life","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":45.148,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":81.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":6144.176,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1386.74,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":null,"release_source":null,"price_in":2.5,"price_out":15.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-high","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-5-medium","name":"GPT-5.5 (medium)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1589.38,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":70.42,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":18.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":49.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":53.982,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":54.514,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":81.25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-04-23","release_source":"epoch","price_in":5.0,"price_out":30.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-sol-unknown-thinking","name":"GPT-5.6 Sol (unknown thinking)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":33.623,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":47.49,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":52.592,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":30.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":76.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":64.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":9619.368,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":null,"release_source":null,"price_in":2.0,"price_out":10.0,"cheapest_provider":"replicate","cheapest_model":"openai/gpt-5.6-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-astra-low","name":"GPT-6 Astra (low)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":85.416664,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":26.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":94.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":67.035,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":87.805,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":62.19,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":54.051,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-09-03","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"azure","cheapest_model":"gpt-6-astra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-astra-medium","name":"GPT-6 Astra (medium)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":92.083336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":29.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":97.07,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":72.788,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":97.561,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":62.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":54.167,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-09-03","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"azure","cheapest_model":"gpt-6-astra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-oss-20b-high","name":"gpt-oss-20b (high)","org":"OpenAI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":1.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":68,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":45.96,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":14.51,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":50.833,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":38.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":40.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2025-08-05","release_source":"epoch","price_in":0.0145,"price_out":0.07,"cheapest_provider":"darkbloom","cheapest_model":"gpt-oss-20b","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-20-0309-reasoning","name":"grok-4.20-0309-reasoning","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":29,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":44.912,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":17.073,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":89.331,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":92.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":30.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-02-17","release_source":"epoch","price_in":1.25,"price_out":2.5,"cheapest_provider":"xai","cheapest_model":"grok-4.20-0309-reasoning","price_observed_at":"2026-11-18T15:47Z"},{"id":"inkling-small-xhigh","name":"Inkling Small (xhigh)","org":"Thinking Machines","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":40.13889,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":46.316,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":17.073,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":88.51,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":90,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":19.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-07-15","release_source":"epoch","price_in":0.45,"price_out":1.2,"cheapest_provider":"deepinfra","cheapest_model":"thinkingmachines/Inkling-Small","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k2-instruct","name":"Kimi K2 Instruct","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":61.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":4.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":81,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":43.8,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":26.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":27.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":39.36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2025-07-11","release_source":"curated","price_in":0.5,"price_out":2.0,"cheapest_provider":"deepinfra","cheapest_model":"moonshotai/Kimi-K2-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k3-low","name":"Kimi K3 (low)","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":12.361111,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":18,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":3.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":84.848,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":68.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":52.662,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2026-07-16","release_source":"epoch","price_in":0.192,"price_out":16.0,"cheapest_provider":"openrouter","cheapest_model":"moonshotai/kimi-k3","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k2-6-thinking","name":"kimi-k2.6-thinking","org":"Moonshot AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":46.919334,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":78.567,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":65.13433,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":64.35825,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":75.141335,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":84.27625,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":79.375,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true}],"n_cells":7,"released_at":null,"release_source":null,"price_in":0.38,"price_out":1.6,"cheapest_provider":"clf_ai_gateway","cheapest_model":"kimi-k2.6","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-4-scout-17b-16e-instruct","name":"Llama-4-Scout-17B-16E-Instruct","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":57.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":51.831,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":11.96,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":7.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":21.296,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2025-04-05","release_source":"epoch","price_in":0.05,"price_out":0.1,"cheapest_provider":"lambda_ai","cheapest_model":"llama-4-scout-17b-16e-instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-large-3","name":"Mistral Large 3","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":264.7,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":65.07,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":16.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":36.574,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":20.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1229.52,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2025-12-02","release_source":"epoch","price_in":0.5,"price_out":1.5,"cheapest_provider":"azure_ai","cheapest_model":"mistral-large-3","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-large-4-preview-unknown","name":"Mistral Large 4 Preview (unknown)","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1213.15,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":10.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":88.24,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":35.79,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":10,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":54.167,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":28.125,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":null,"release_source":null,"price_in":0.7,"price_out":2.1,"cheapest_provider":"azure_ai","cheapest_model":"mistral-large-4-latest","price_observed_at":"2026-11-18T15:47Z"},{"id":"o1-medium","name":"o1 (medium)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":83.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":10.175,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":75.758,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":38.832,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":73.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":36.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2024-12-05","release_source":"curated","price_in":15.0,"price_out":60.0,"cheapest_provider":"azure","cheapest_model":"o1","price_observed_at":"2026-11-18T15:47Z"},{"id":"o3-mini-medium","name":"o3-mini (medium)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":2.08,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":11,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":22.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":50,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":10.526,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":74.274,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":63.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2025-01-31","release_source":"curated","price_in":1.1,"price_out":4.4,"cheapest_provider":"azure","cheapest_model":"o3-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-30b-a3b","name":"Qwen3-30B-A3B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":60.27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":134.18,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":40.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":61.679,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":15.82,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":62.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":29.75,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":"2025-04-28","release_source":"epoch","price_in":0.08,"price_out":0.25,"cheapest_provider":"gmi","cheapest_model":"Qwen/Qwen3-30B-A3B","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-30b-a3b-thinking-2507","name":"qwen3-30b-a3b-thinking-2507","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":69.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":72.22222,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":19.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":80.9,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":70.278,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":32.986,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":7,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-5-35b-a3b","name":"qwen3.5-35b-a3b","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":221.8,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":80,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":144.08,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":83.46,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":29.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":54.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"fresh, decontaminated GitHub-issue tasks (resolved %)","label":"SWE-rebench","trust":2,"value":17.1,"score_max":100,"source_url":"https://swe-rebench.com/","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_rebench","source_label":"SWE-rebench leaderboard","higher_is_better":true}],"n_cells":7,"released_at":"2026-02-24","release_source":"epoch","price_in":0.057,"price_out":0.459,"cheapest_provider":"empiriolabs","cheapest_model":"qwen3-5-35b-a3b","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-6-plus","name":"qwen3.6-plus","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"LiveBench release 2026_06_25","label":"LiveBench Agentic Coding","trust":4,"value":41.363667,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_agentic_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Coding","trust":4,"value":78.1845,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_coding","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Data Analysis","trust":4,"value":69.91167,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_data_analysis","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench IF","trust":4,"value":58.342,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_if","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Language","trust":4,"value":74.989334,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_language","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Mathematics","trust":4,"value":83.72475,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_mathematics","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true},{"note":"LiveBench release 2026_06_25","label":"LiveBench Reasoning","trust":4,"value":75.827,"score_max":100,"source_url":"https://raw.githubusercontent.com/LiveBench/livebench.github.io/main/public/table_2026_06_25.csv","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"livebench_reasoning","source_label":"LiveBench 2026_06_25 official CSV","higher_is_better":true}],"n_cells":7,"released_at":"2026-04-02","release_source":"openrouter","price_in":0.325,"price_out":1.95,"cheapest_provider":"openrouter","cheapest_model":"qwen/qwen3.6-plus","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-5-sonnet-20241022","name":"Claude 3.5-Sonnet-20241022","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":84.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":51.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"Aider official benchmark","label":"Aider Refactoring","trust":4,"value":92.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/refactor.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_refactor","source_label":"Aider Aider Refactoring leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":4,"value":30,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Test","trust":2,"value":24.89,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_test","source_label":"SWE-bench Test leaderboard (unverified)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":51.6,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":6,"released_at":"2024-10-22","release_source":"name","price_in":3.0,"price_out":15.0,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"anthropic/claude-3-5-sonnet-20241022","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-xhigh","name":"Claude Fable 5 (xhigh)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":88.333336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":97.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":69.912,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":61.02,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":70.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":15.587,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-06-09","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-1-low","name":"Claude Fable 5.1 (low)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":78.333336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":27.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":45.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":94.13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":63.46,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":56.713,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-09-01","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5-1","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-2-low","name":"Claude Fable 5.2 (Low)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":86.666664,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":29.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":53.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":98.13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":69.63,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":57.755,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-11-09","release_source":"epoch","price_in":8.0,"price_out":40.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5-2","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-2-medium","name":"Claude Fable 5.2 (Medium)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":89.583336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":30.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":54.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":98.13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":70.03,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":59.722,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-11-09","release_source":"epoch","price_in":8.0,"price_out":40.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5-2","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-haiku-5-5-medium","name":"Claude Haiku 5.5 (medium)","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":9.444445,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":12.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":36.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":84.27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":45.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":48.958,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-10-07","release_source":"epoch","price_in":0.1,"price_out":0.5,"cheapest_provider":"anthropic","cheapest_model":"claude-haiku-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-7","name":"Claude Opus 4.7","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":1000000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":157.5,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1555,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1495,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1233,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1494,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1298,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":6,"released_at":"2026-04-16","release_source":"epoch","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-7","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-7-no-thinking","name":"Claude Opus 4.7 (no thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1323.05,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing working exploits","label":"ExploitBench","trust":2,"value":26.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"exploitbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":44.12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1557.37,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":76.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-04-16","release_source":"epoch","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-7","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-5-5-low","name":"Claude Opus 5.5 (Low)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":70.138885,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":17.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":43.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":95.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":64.92,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":58.565,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-09-22","release_source":"curated","price_in":4.0,"price_out":20.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-6","name":"Claude Sonnet 4.6","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":1000000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":152.86,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1521,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1482,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1472,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1275,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"tool use across 1,000 MCP tasks","label":"MCP Atlas","trust":4,"value":69.5,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/mcp_atlas","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"mcp_atlas","source_label":"Scale Labs MCP Atlas leaderboard","higher_is_better":true}],"n_cells":6,"released_at":"2026-02-17","release_source":"epoch","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-6-medium","name":"Claude Sonnet 4.6 (medium)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1327.3,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":83.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks","label":"OSWorld 2","trust":2,"value":9.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"osworld_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":82.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":66.07,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-02-17","release_source":"epoch","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-6-no-thinking","name":"Claude Sonnet 4.6 (no thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"writing working exploits","label":"ExploitBench","trust":2,"value":23.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"exploitbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"computer-use tasks (v1)","label":"OSWorld","trust":2,"value":72.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"os_world","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":75.207,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":53.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1520.99,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-02-17","release_source":"epoch","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-5-low","name":"Claude Sonnet 5 (low)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":4.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":24.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":73.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":30.512,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":49.65,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":50.116,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-06-30","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"aihubmix","cheapest_model":"claude-sonnet-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-5-5-low","name":"Claude Sonnet 5.5 (low)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":24.027779,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":11.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":35.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":93.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":64.54,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":49.074,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-09-28","release_source":"curated","price_in":2.0,"price_out":10.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-haiku-20240307","name":"claude-3-haiku-20240307","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":47.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":50.13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":36.301,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":8.84,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":1.806,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":9.84,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2024-03-13","release_source":"curated","price_in":0.25,"price_out":1.25,"cheapest_provider":"vertex_ai","cheapest_model":"claude-3-haiku@20240307","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-flash-high","name":"DeepSeek v4 Flash (high)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":678.2,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":3.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1581,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":42.014,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1430.93,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":43.76,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-04-24","release_source":"epoch","price_in":0.09,"price_out":0.18,"cheapest_provider":"deepinfra","cheapest_model":"deepseek-ai/DeepSeek-V4-Flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-1-flash-unknown","name":"DeepSeek V4.1 Flash (unknown)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":39.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":89.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":46.97,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":54,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":9.58,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":66.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-09-09","release_source":"epoch","price_in":0.01,"price_out":1.2,"cheapest_provider":"coralbricks","cheapest_model":"deepseek-v4.1-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v3-2-exp","name":"DeepSeek-V3.2-Exp","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":74.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":144.39,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":52.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"work tasks in a simulated software company","label":"TheAgentCompany","trust":2,"value":42.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"agent_company","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":1034.004,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1271.75,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":null,"release_source":null,"price_in":0.27,"price_out":0.41,"cheapest_provider":"gmi","cheapest_model":"deepseek-ai/DeepSeek-V3.2-Exp","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-0-flash-thinking-exp","name":"Gemini 2.0 Flash Thinking Exp","org":"Google DeepMind,Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":1.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":52.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":57.071,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":6.56,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":57.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":30.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-5-pro-exp-mar-2025","name":"Gemini 2.5 Pro Exp (Mar 2025)","org":"Google DeepMind","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":43.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":4.14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":66.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":83.838,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":18.16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":51.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2025-03-25","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-3-flash","name":"Gemini 3 Flash","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":152.34,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1413,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1473,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1273,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Multilingual","trust":4,"value":72.7,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multilingual","source_label":"SWE-bench Multilingual leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":75.8,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":6,"released_at":"2025-12-17","release_source":"curated","price_in":0.5,"price_out":3.0,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3-flash-preview","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-7-flash-low","name":"Gemini 3.7 Flash (low)","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":52.916668,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":5.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":93.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":53.761,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":47.09,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":53.588,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-08-13","release_source":"epoch","price_in":0.75,"price_out":3.75,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3.7-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemma-3-12b-it","name":"gemma-3-12b-it","org":"Google DeepMind","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":48.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":39.457,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":4.46,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":16.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":16.435,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2025-03-12","release_source":"epoch","price_in":0.05,"price_out":0.1,"cheapest_provider":"novita","cheapest_model":"google/gemma-3-12b-it","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-4-6","name":"GLM-4.6","org":"Z.ai (Zhipu AI),Tsinghua University","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":340.82,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":1.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":55.4,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":38.426,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":24.532,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1340.6,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2025-09-30","release_source":"curated","price_in":0.45,"price_out":1.8,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"zai/glm-4.6","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4o","name":"GPT-4o","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":128000,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"Aider official benchmark","label":"Aider Refactoring","trust":4,"value":62.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/refactor.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_refactor","source_label":"Aider Aider Refactoring leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":4,"value":24.67,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Multimodal","trust":4,"value":30.37,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multimodal","source_label":"SWE-bench Multimodal leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Test","trust":4,"value":14.6,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_test","source_label":"SWE-bench Test leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":26.2,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":6,"released_at":"2024-05-13","release_source":"curated","price_in":2.5,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-4o","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-mini-minimal","name":"GPT-5 mini (minimal)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.67,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":5.965,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":71.717,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":10,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":55.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2025-08-07","release_source":"curated","price_in":0.25,"price_out":2.0,"cheapest_provider":"azure","cheapest_model":"gpt-5-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-nano-medium","name":"GPT-5 nano (medium)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0.88,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":44.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":67.424,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":74.167,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":11.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2025-08-07","release_source":"curated","price_in":0.05,"price_out":0.4,"cheapest_provider":"azure","cheapest_model":"gpt-5-nano","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-nano-minimal","name":"GPT-5 nano (minimal)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":1.754,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":48.485,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":35.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2025-08-07","release_source":"curated","price_in":0.05,"price_out":0.4,"cheapest_provider":"azure","cheapest_model":"gpt-5-nano","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-1-no-thinking","name":"GPT-5.1 (no thinking)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":1.94,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":66.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":6.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":15,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":37.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":null,"release_source":null,"price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"global/gpt-5.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4-low","name":"GPT-5.4 (low)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":29.17,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":19,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":35.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":84.848,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":84.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-03-05","release_source":"epoch","price_in":2.5,"price_out":15.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-high","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4-none","name":"GPT-5.4 (none)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1086.03,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":74.747,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":57.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":57.44,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-03-05","release_source":"epoch","price_in":2.5,"price_out":15.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-high","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4-mini-none","name":"GPT-5.4 mini (none)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":17.193,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":64.141,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":11,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":26.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":37.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-03-17","release_source":"epoch","price_in":0.75,"price_out":4.5,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4-nano-low","name":"GPT-5.4 nano (low)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.53,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":20,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":20.351,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":72.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":68.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-03-17","release_source":"epoch","price_in":0.2,"price_out":1.25,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-nano","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4-nano-no-thinking","name":"GPT-5.4 nano (no thinking)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":4.561,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":55.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":46.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":37.96,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":null,"release_source":null,"price_in":0.2,"price_out":1.25,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-nano","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-luna-unknown-thinking","name":"GPT-5.6 Luna (unknown thinking)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":43,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":22.56,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":39.81,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":22.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":46.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":4094.712,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":null,"release_source":null,"price_in":0.2,"price_out":1.2,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.6-luna","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-5","name":"Grok 4.5","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":154.97,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1552,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1452,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1213,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1466,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1279,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true}],"n_cells":6,"released_at":"2026-07-08","release_source":"epoch","price_in":2.0,"price_out":6.0,"cheapest_provider":"github_copilot","cheapest_model":"grok-4.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-6-unknown","name":"Grok 4.6 (unknown)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":65.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":33.189,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":48.01,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":51,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":75.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":9047.033,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-08-12","release_source":"curated","price_in":2.0,"price_out":6.0,"cheapest_provider":"aihubmix","cheapest_model":"grok-4.6","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-1-fast-reasoning","name":"grok-4-1-fast-reasoning","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":394.93,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":87.73,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":56,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":1106.628,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1223,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":null,"release_source":null,"price_in":0.2,"price_out":0.5,"cheapest_provider":"azure_ai","cheapest_model":"grok-4-1-fast-reasoning","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-fast","name":"grok-4-fast","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":30,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":82.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":144.35,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":94.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1149,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":42.86,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"inkling-small","name":"Inkling Small","org":"Thinking Machines","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":8.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":150.78,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":89.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":49.653,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1406.58,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-07-15","release_source":"epoch","price_in":0.45,"price_out":1.2,"cheapest_provider":"deepinfra","cheapest_model":"thinkingmachines/Inkling-Small","price_observed_at":"2026-11-18T15:47Z"},{"id":"mimo-v2-flash","name":"MiMo-V2-Flash","org":"XiaomiMiMo","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":737.95,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":83.333336,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":84.9,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":25.926,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1330.64,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":null,"release_source":null,"price_in":0.11,"price_out":0.33,"cheapest_provider":"novita","cheapest_model":"xiaomimimo/mimo-v2-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"minimax-m2","name":"MiniMax-M2","org":"MiniMax","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":82,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":61,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":30.037,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":160.596,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1297.78,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2025-10-27","release_source":"curated","price_in":0.255,"price_out":1.02,"cheapest_provider":"openrouter","cheapest_model":"minimax/minimax-m2","price_observed_at":"2026-11-05T12:47Z"},{"id":"minimax-m2-1","name":"MiniMax-M2.1","org":"MiniMax","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":623.83,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":80.80808,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":88,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":34.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":36.629,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1387.48,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2025-12-23","release_source":"epoch","price_in":0.3,"price_out":1.2,"cheapest_provider":"bedrock","cheapest_model":"minimax.minimax-m2.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-medium-2508","name":"mistral-medium-2508","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":210.18,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":67.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":19.05,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":32.407,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":33.13,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2025-08-12","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"muse-spark-1-3-unknown","name":"Muse Spark 1.3 (unknown)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":58.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":96.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":27.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":53.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":55,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":81.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2026-09-02","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"empiriolabs","cheapest_model":"muse-spark-1-3","price_observed_at":"2026-11-18T15:47Z"},{"id":"o4-mini-low","name":"o4-mini (low)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.67,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":15,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":16.14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":75.253,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":57.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":19.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2025-04-16","release_source":"curated","price_in":1.0,"price_out":4.0,"cheapest_provider":"replicate","cheapest_model":"openai/o4-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"phi-4","name":"phi-4","org":"Microsoft Research","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":11.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":129.24,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":56.061,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":70.4,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":13.75,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2024-12-12","release_source":"epoch","price_in":0.07,"price_out":0.14,"cheapest_provider":"deepinfra","cheapest_model":"microsoft/phi-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-coder-480b-a35b-instruct","name":"Qwen3-Coder-480B-A35B-Instruct","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":461.45,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":4.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":55.4,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":27.191,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1273.75,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":41.17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":6,"released_at":"2025-07-22","release_source":"curated","price_in":0.38,"price_out":1.55,"cheapest_provider":"novita","cheapest_model":"qwen/qwen3-coder-480b-a35b-instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-5-sonnet","name":"Claude 3.5 Sonnet","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":130,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":4,"value":39,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Multimodal","trust":4,"value":25.34,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multimodal","source_label":"SWE-bench Multimodal leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Test","trust":4,"value":18.13,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_test","source_label":"SWE-bench Test leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":33.6,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":5,"released_at":"2024-06-20","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"snowflake","cheapest_model":"claude-3-5-sonnet","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-7-sonnet","name":"Claude 3.7 Sonnet","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":141.05,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":4,"value":48,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Multimodal","trust":4,"value":31.33,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multimodal","source_label":"SWE-bench Multimodal leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Test","trust":4,"value":33.83,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_test","source_label":"SWE-bench Test leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":52.8,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":5,"released_at":"2025-02-24","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"replicate","cheapest_model":"anthropic/claude-3.7-sonnet","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-medium","name":"Claude Fable 5 (medium)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":82.5,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":95.73,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":65.367,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":60.32,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":4339.813,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-06-09","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-5-16k-thinking","name":"Claude Opus 4.5 (16k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1025.38,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":85.48,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":288.905,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":81.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":63.74,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2025-11-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-5-high","name":"Claude Opus 4.5 (high)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":89.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":54.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":44.46,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"tool use across 1,000 MCP tasks","label":"MCP Atlas","trust":4,"value":69.8,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/mcp_atlas","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"mcp_atlas","source_label":"Scale Labs MCP Atlas leaderboard","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":6.515,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2025-11-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-6","name":"Claude Opus 4.6","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":1000000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":156.35,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1537,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1507,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1497,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1294,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":5,"released_at":"2026-02-05","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-7-xhigh","name":"Claude Opus 4.7 (xhigh)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":90.152,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":97.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":24.34,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":51.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-04-16","release_source":"epoch","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-7","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-8-high","name":"Claude Opus 4.8 (High)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":72.08,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1556,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1475,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1482,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1286,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":5,"released_at":"2026-05-28","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-8","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-5","name":"Claude Opus 5","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":1000000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":164.74,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":92.929,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":37,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":97.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":86.34,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-07-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"aihubmix","cheapest_model":"claude-opus-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-5-unknown","name":"Claude Opus 5 (unknown)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":30.369,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":79.599,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":35.66,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":80.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":11181.873,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-07-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"aihubmix","cheapest_model":"claude-opus-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-5-5-unknown","name":"Claude Opus 5.5 (unknown)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":51.193,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":90.555,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":21.25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":88.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":9235.248,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-09-22","release_source":"curated","price_in":4.0,"price_out":20.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-6-unknown-thinking","name":"Claude Sonnet 4.6 (unknown thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":6.725,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":24.31,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":48.757,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":53.427,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":7204.14,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-02-17","release_source":"epoch","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-sonnet-20240229","name":"claude-3-sonnet-20240229","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":54.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":53.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":40.593,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":2.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":10.16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2024-03-04","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"vertex_ai","cheapest_model":"claude-3-sonnet@20240229","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-haiku-4-5-20251001-32k","name":"claude-haiku-4-5-20251001_32K","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":653.48,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":71.212,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":66.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":12.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2025-10-15","release_source":"curated","price_in":1.0,"price_out":5.0,"cheapest_provider":"anthropic","cheapest_model":"claude-haiku-4-5-20251001","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-1-20250805-16k","name":"claude-opus-4-1-20250805_16K","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":674.77,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":77.273,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":113.685,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":64.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":45.86,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2025-08-05","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-v4-flash-max","name":"DeepSeek v4 Flash (max)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":7.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":86.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":35.88,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":44.907,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":45.63,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-04-24","release_source":"epoch","price_in":0.09,"price_out":0.18,"cheapest_provider":"deepinfra","cheapest_model":"deepseek-ai/DeepSeek-V4-Flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-pro-0813-high","name":"DeepSeek V4 Pro 0813 (high)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":59.72222,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":18.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":28.55,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":51.736,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1580.53,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-08-13","release_source":"epoch","price_in":0.111,"price_out":4.44,"cheapest_provider":"openrouter","cheapest_model":"deepseek/deepseek-v4-pro-0813","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v3-1-terminus","name":"DeepSeek-V3.1-Terminus","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":745.17,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":1.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":81.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":28.58,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":37.963,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2025-09-22","release_source":"epoch","price_in":0.27,"price_out":1.0,"cheapest_provider":"deepinfra","cheapest_model":"deepseek-ai/DeepSeek-V3.1-Terminus","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-6-flash-high","name":"Gemini 3.6 Flash (High)","org":"Google","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":60.416668,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1536,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1456,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1483,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1280,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true}],"n_cells":5,"released_at":"2026-07-21","release_source":"epoch","price_in":0.75,"price_out":3.75,"cheapest_provider":"gemini","cheapest_model":"gemini-3.6-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-8-flash-low","name":"Gemini 3.8 Flash (low)","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":77.5,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":94.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":51.16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":54.977,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-09-02","release_source":"epoch","price_in":0.75,"price_out":3.75,"cheapest_provider":"gemini","cheapest_model":"gemini-3.8-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-4-argon-low","name":"Gemini 4 Argon (low)","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1705.4,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":42.083332,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":50.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":97.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":62.92,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-09-30","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"gemini","cheapest_model":"gemini-4-argon","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-1-5-flash-002","name":"gemini-1.5-flash-002","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":14.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":53.79,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":47.317,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":16.25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":24.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2024-09-24","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-5-flash-lite-preview-06-17-default-thinking-length","name":"gemini-2.5-flash-lite-preview-06-17 (Default thinking length)","org":"Google DeepMind","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":325.9,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":62.83,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":47.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":18.07,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":35.22,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2025-06-17","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-5-flash-preview-05-20","name":"gemini-2.5-flash-preview-05-20","org":"Google DeepMind","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":55.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":2.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":77.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":10.96,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":70.833,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2025-05-20","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"glm-5-3-unknown-thinking","name":"GLM-5.3 (unknown thinking)","org":"Z.ai (Zhipu AI)","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":56.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":87.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":55.51,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":66.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":8163.61,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":null,"release_source":null,"price_in":0.0699,"price_out":1.4,"cheapest_provider":"openrouter","cheapest_model":"z-ai/glm-5.3","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-low","name":"GPT-5 (low)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.94,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":49.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":37.193,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2025-08-07","release_source":"curated","price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-unknown-thinking","name":"GPT-5 (unknown thinking)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":10.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":25.32,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":1.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":42.94,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":49.607,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":null,"release_source":null,"price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-mini-unknown-thinking","name":"GPT-5 mini (unknown thinking)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":8.193,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":19.44,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":39.236,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":34.831,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":null,"release_source":null,"price_in":0.25,"price_out":2.0,"cheapest_provider":"azure","cheapest_model":"gpt-5-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-1-low","name":"GPT-5.1 (low)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":42.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":19,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":63.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1359,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2025-11-12","release_source":"curated","price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"global/gpt-5.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-1-medium","name":"GPT-5.1 (medium)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":85.038,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":85.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":47.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1392.03,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2025-11-12","release_source":"curated","price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"global/gpt-5.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-2","name":"GPT-5.2","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":272000,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0.83,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":154.52,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1401,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Multilingual","trust":4,"value":66.7,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multilingual","source_label":"SWE-bench Multilingual leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":72.8,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":5,"released_at":"2025-12-11","release_source":"curated","price_in":1.75,"price_out":14.0,"cheapest_provider":"azure","cheapest_model":"gpt-5.2","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-2-none","name":"GPT-5.2 (none)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":73.232,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":62.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":49.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2025-12-11","release_source":"curated","price_in":1.75,"price_out":14.0,"cheapest_provider":"azure","cheapest_model":"gpt-5.2","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-3-codex","name":"GPT-5.3 Codex","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":272000,"open_weight":false,"cells":[{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":349.531,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":78.427,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":5940.124,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1369.86,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":79.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-02-05","release_source":"epoch","price_in":1.75,"price_out":14.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.3-codex","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-terra-unknown-thinking","name":"GPT-5.6 Terra (unknown thinking)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":30.803,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":41.31,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":24.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":48.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":7343.206,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":null,"release_source":null,"price_in":2.0,"price_out":12.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.6-terra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-luna-xhigh","name":"GPT-6 Luna (XHigh)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1576.92,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":41.944443,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":17.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":51.736,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":7.874,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-09-22","release_source":"epoch","price_in":0.1,"price_out":0.5,"cheapest_provider":"azure","cheapest_model":"gpt-6-luna","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-1-sol-unknown-thinking","name":"GPT-6.1 Sol (unknown thinking)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":43.167,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"playing Game Boy Advance games","label":"GBA-Eval","trust":2,"value":81.911,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gbaeval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":99,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":82.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":12860.612,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":null,"release_source":null,"price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.1-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-oss-20b","name":"gpt-oss-20b","org":"OpenAI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":566.05,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":134.88,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":71.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":73.6,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":3.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2025-08-05","release_source":"epoch","price_in":0.0145,"price_out":0.07,"cheapest_provider":"darkbloom","cheapest_model":"gpt-oss-20b","price_observed_at":"2026-11-18T15:47Z"},{"id":"granite-4-1-30b","name":"granite-4.1-30b","org":"ibm-granite","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":45.76,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":94.16,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":64.09,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":25.81,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-04-29","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"grok-4-6-low","name":"Grok 4.6 (low)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":27.63889,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":5.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":33.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":41.648,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":49.421,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-08-12","release_source":"curated","price_in":2.0,"price_out":6.0,"cheapest_provider":"aihubmix","cheapest_model":"grok-4.6","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-6-medium","name":"Grok 4.6 (medium)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":61.25,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":17.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":36.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-horizon software engineering","label":"DeepSWE","trust":2,"value":67.478,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":55.903,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-08-12","release_source":"curated","price_in":2.0,"price_out":6.0,"cheapest_provider":"aihubmix","cheapest_model":"grok-4.6","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-7-unknown","name":"Grok 4.7 (unknown)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":70,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":32.538,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":47.59,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":10536.832,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-09-21","release_source":"curated","price_in":2.0,"price_out":6.0,"cheapest_provider":"azure_ai","cheapest_model":"grok-4.7","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-2-1212","name":"grok-2-1212","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":65.21,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":53.788,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":11.528,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":22.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":22.24,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2024-12-12","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"grok-3-beta","name":"grok-3-beta","org":"xAI","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":53.3,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":29.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":58.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":75.758,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":55.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"kimi-k3-high","name":"Kimi K3 (high)","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":55,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":37,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":91.919,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":93.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":7.544,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-07-16","release_source":"epoch","price_in":0.192,"price_out":16.0,"cheapest_provider":"openrouter","cheapest_model":"moonshotai/kimi-k3","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-3-2-90b-vision-instruct","name":"Llama-3.2-90B-Vision-Instruct","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":27.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":0.38,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":46.085857,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":93.10084,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":2.639,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2024-09-24","release_source":"epoch","price_in":2.0,"price_out":2.0,"cheapest_provider":"watsonx","cheapest_model":"meta-llama/llama-3-2-90b-vision-instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"magistral-small-1-2","name":"Magistral Small 1.2","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":61.31,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":47.601,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":28.056,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":35.185,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2025-09-18","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mercury-2","name":"Mercury 2","org":"Inception Labs","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":785.58,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.849,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":38.657,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1166.59,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":43.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-03-04","release_source":"openrouter","price_in":0.25,"price_out":0.75,"cheapest_provider":"inception","cheapest_model":"mercury-2","price_observed_at":"2026-11-18T15:47Z"},{"id":"mimo-v2-5","name":"mimo-v2.5","org":"Xiaomi Corp","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":513.95,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":3.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":43.056,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1437.16,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-04-22","release_source":"openrouter","price_in":0.119,"price_out":0.238,"cheapest_provider":"gmi","cheapest_model":"XiaomiMiMo/MiMo-V2.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"mimo-v2-6-flash","name":"MiMo-V2.6 Flash","org":"Xiaomi Corp","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1244.2,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1636,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":63,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":51.273,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-09-21","release_source":"epoch","price_in":0.112,"price_out":0.224,"cheapest_provider":"gmi","cheapest_model":"XiaomiMiMo/MiMo-V2.6-Flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-small-2503","name":"mistral-small-2503","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":58.64,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":47.475,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":5.833,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":27.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2025-03-17","release_source":"epoch","price_in":0.1,"price_out":0.3,"cheapest_provider":"azure_ai","cheapest_model":"mistral-small-2503","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark-1-2","name":"Muse Spark 1.2","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":45.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":155.06,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":43,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":74.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-08-05","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"empiriolabs","cheapest_model":"muse-spark-1-2","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-wonder-unknown","name":"Muse Wonder (unknown)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":31.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":95.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":56.84,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":84,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":9432.18,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-10-12","release_source":"epoch","price_in":4.0,"price_out":20.0,"cheapest_provider":"openrouter","cheapest_model":"meta/muse-wonder","price_observed_at":"2026-11-18T15:47Z"},{"id":"o1-mini-medium","name":"o1-mini (medium)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":10,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":59.501,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":44.722,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":18.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":36.32,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2024-09-12","release_source":"curated","price_in":1.1,"price_out":4.4,"cheapest_provider":"azure","cheapest_model":"o1-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"o3-low","name":"o3 (low)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.99,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":19.298,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":79.798,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":60,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2025-04-16","release_source":"curated","price_in":2.0,"price_out":8.0,"cheapest_provider":"azure","cheapest_model":"o3","price_observed_at":"2026-11-18T15:47Z"},{"id":"o3-mini-2025-01-31-low","name":"o3-mini-2025-01-31 (low)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":3.86,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":68.182,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":1.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":44.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2025-01-31","release_source":"name","price_in":1.1,"price_out":4.4,"cheapest_provider":"azure","cheapest_model":"o3-mini-2025-01-31","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen2-72b-instruct","name":"qwen2-72b-instruct","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":55.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":40.783,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":2.244,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"work tasks in a simulated software company","label":"TheAgentCompany","trust":2,"value":1.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"agent_company","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":11.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2024-06-07","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-7b-instruct","name":"qwen2.5-7b-instruct","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":7.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":47.73,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":35.48,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":6.42,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":2.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2024-09-19","release_source":"epoch","price_in":0.04,"price_out":0.1,"cheapest_provider":"deepinfra","cheapest_model":"Qwen/Qwen2.5-7B-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-5-122b-a10b-none","name":"qwen3.5-122b-a10b (none)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":84.27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":32.23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":35.648,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-02-24","release_source":"epoch","price_in":0.1126,"price_out":0.9008,"cheapest_provider":"aihubmix","cheapest_model":"qwen3.5-122b-a10b","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-8-max-unknown","name":"Qwen3.8 Max (unknown)","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":63.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":20,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":58,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":53.241,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1670.58,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2026-08-02","release_source":"epoch","price_in":1.65,"price_out":4.951,"cheapest_provider":"deepinfra","cheapest_model":"Qwen/Qwen3.8-Max","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwq-32b","name":"QwQ-32B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":20.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":136.48,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":83.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":65.341,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":59.167,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":5,"released_at":"2025-03-06","release_source":"curated","price_in":0.15,"price_out":0.4,"cheapest_provider":"deepinfra","cheapest_model":"Qwen/QwQ-32B","price_observed_at":"2026-11-18T15:47Z"},{"id":"c4ai-command-a-03-2025","name":"c4ai-command-a-03-2025","org":"Cohere","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":61.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":50.50505,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":10.28,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2025-03-13","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"c4ai-command-r-plus-08-2024","name":"c4ai-command-r-plus-08-2024","org":"Cohere,Cohere Labs (formerly Cohere for AI)","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":54.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":34.343433,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":5.01,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":17.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2024-08-30","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-2","name":"Claude 2","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":117.12,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":4,"value":3,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Test","trust":4,"value":1.96,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_test","source_label":"SWE-bench Test leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":4.4,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":4,"released_at":"2023-07-11","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-3-opus","name":"Claude 3 Opus","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":125.58,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":4,"value":11.67,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Test","trust":4,"value":10.51,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_test","source_label":"SWE-bench Test leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":15.8,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":4,"released_at":"2024-03-04","release_source":"curated","price_in":15.0,"price_out":75.0,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"anthropic/claude-3-opus","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-7-sonnet-unknown-thinking","name":"Claude 3.7 Sonnet (unknown thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":4.23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":3.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":8.04,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":46.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2025-02-24","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"replicate","cheapest_model":"anthropic/claude-3.7-sonnet","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5","name":"Claude Fable 5","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":1000000,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":63.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":164.21,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":16.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":4529.944,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-06-09","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-haiku-5-5-unknown","name":"Claude Haiku 5.5 (unknown)","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":32.972,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":30.12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":59.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":3602.463,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-10-07","release_source":"epoch","price_in":0.1,"price_out":0.5,"cheapest_provider":"anthropic","cheapest_model":"claude-haiku-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4","name":"Claude Sonnet 4","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.27,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":141.88,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Multimodal","trust":4,"value":34.43,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multimodal","source_label":"SWE-bench Multimodal leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":74.8,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":4,"released_at":"2025-05-22","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"gmi","cheapest_model":"anthropic/claude-sonnet-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-5-16k-thinking","name":"Claude Sonnet 4.5 (16k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":78.788,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":121.947,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":71.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":47.71,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2025-09-29","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-5-59k-thinking","name":"Claude Sonnet 4.5 (59k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":10,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":82.323,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":77.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":30.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2025-09-29","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-2-1","name":"claude-2.1","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":50.95,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":32.955,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":1.944,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":7.06,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2023-11-21","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-sonnet-4-20250514-16k","name":"claude-sonnet-4-20250514_16K","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":75.758,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":74.942,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":53.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":46.11,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2025-05-14","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-v4-pro-0813-unknown","name":"DeepSeek V4 Pro 0813 (unknown)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":47.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":93.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":45.45,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":50,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-08-13","release_source":"epoch","price_in":0.111,"price_out":4.44,"cheapest_provider":"openrouter","cheapest_model":"deepseek/deepseek-v4-pro-0813","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-1-flash-high","name":"DeepSeek V4.1 Flash (high)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":67.5,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":43.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":46.25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":5.946,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-09-09","release_source":"epoch","price_in":0.01,"price_out":1.2,"cheapest_provider":"coralbricks","cheapest_model":"deepseek-v4.1-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-chat","name":"deepseek-chat","org":"DeepSeek","color":"#64748b","category":"frontier","params_b":null,"context_window":131072,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":62.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":71.212,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":15.24,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":48.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2025-12-01","release_source":"epoch","price_in":0.19,"price_out":0.38,"cheapest_provider":"inferenceapis","cheapest_model":"deepseek-chat","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v3-1","name":"DeepSeek-V3.1","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":140.8,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":52.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":40,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":37.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2025-08-21","release_source":"epoch","price_in":0.25,"price_out":0.95,"cheapest_provider":"deepinfra","cheapest_model":"deepseek-ai/DeepSeek-V3.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v3-2","name":"DeepSeek-V3.2","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":164000,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":4.03,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":146.26,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Multilingual","trust":4,"value":59,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multilingual","source_label":"SWE-bench Multilingual leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":70,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":4,"released_at":"2025-12-01","release_source":"curated","price_in":0.26,"price_out":0.38,"cheapest_provider":"deepinfra","cheapest_model":"deepseek-ai/DeepSeek-V3.2","price_observed_at":"2026-11-18T15:47Z"},{"id":"doubao-seed-2-0-lite-apr-2026-high","name":"Doubao Seed 2.0 Lite (Apr 2026, high)","org":"ByteDance","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":20,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":88.131,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":91.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":31.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-04-28","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"doubao-seed-2-1-pro-high","name":"Doubao Seed 2.1 Pro (high)","org":"ByteDance","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":21,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":89.331,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":95.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":44.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-06-23","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"doubao-seed-2-1-turbo-high","name":"Doubao Seed 2.1 Turbo (high)","org":"ByteDance","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":89.583,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":96.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":29.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-06-23","release_source":"epoch","price_in":0.46475,"price_out":2.32375,"cheapest_provider":"aihubmix","cheapest_model":"doubao-seed-2-1-turbo","price_observed_at":"2026-11-18T15:47Z"},{"id":"doubao-seed-evolving-nov-2026-high","name":"Doubao Seed Evolving (Nov 2026, high)","org":"ByteDance","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":40,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":93.119,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":99.167,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":41.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-11-02","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"doubao-seed-1-6-thinking","name":"Doubao-Seed-1.6 (thinking)","org":"ByteDance","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":77.588,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":69.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":17.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"doubao-seed-2-0-pro-high","name":"Doubao-Seed-2.0 Pro (high)","org":"ByteDance","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":20,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":88.194,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":93.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"short factual questions","label":"SimpleQA Verified","trust":2,"value":39.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simpleqa_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-02-14","release_source":"epoch","price_in":0.4822,"price_out":2.411,"cheapest_provider":"aihubmix","cheapest_model":"doubao-seed-2-0-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-1-5-flash-may-2024","name":"Gemini 1.5 Flash (May 2024)","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":51.38,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":120.55,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":40.372,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":3.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2024-05-23","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-3-7-flash-unknown","name":"Gemini 3.7 Flash (unknown)","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":67.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":43.59,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":58,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"paid freelance projects completed","label":"Remote Labor Index","trust":2,"value":5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"rli","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-08-13","release_source":"epoch","price_in":0.75,"price_out":3.75,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3.7-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-4-carbon-unknown","name":"Gemini 4 Carbon (unknown)","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":84.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":58.568,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":92.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":16484.207,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-10-28","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"gemini","cheapest_model":"gemini-4-carbon","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-1-5-pro-001","name":"gemini-1.5-pro-001","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":57.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":56.47,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":45.865,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":6.806,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2024-05-14","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemma-3-4b-it","name":"gemma-3-4b-it","org":"Google DeepMind","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":50.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":23.232,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":2.78,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":7.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2025-03-12","release_source":"epoch","price_in":0.05,"price_out":0.1,"cheapest_provider":"deepinfra","cheapest_model":"google/gemma-3-4b-it","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-4-5","name":"glm-4.5","org":"Z.ai (Zhipu AI),Tsinghua University","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":344.82,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":8.32,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":84.6,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":54.2,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":4,"released_at":"2025-07-28","release_source":"curated","price_in":0.4,"price_out":1.6,"cheapest_provider":"deepinfra","cheapest_model":"zai-org/GLM-4.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-4-5-air","name":"GLM-4.5-Air","org":"Z.ai (Zhipu AI),Tsinghua University","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":71.71717,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"software performance optimisation","label":"GSO-Bench","trust":2,"value":2.94,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gso","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":8.12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":81.4,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":4,"released_at":"2025-07-20","release_source":"epoch","price_in":0.125,"price_out":0.45,"cheapest_provider":"pinstripes","cheapest_model":"ps/glm-4.5-air","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-5-3-flash","name":"GLM-5.3-Flash","org":"Z.ai (Zhipu AI)","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":153.78,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1616,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-07","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-06)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1473,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1278,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true}],"n_cells":4,"released_at":"2026-08-20","release_source":"epoch","price_in":0.06,"price_out":0.2,"cheapest_provider":"clf_ai_gateway","cheapest_model":"glm-5.3-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-5-3-flash-unknown","name":"GLM-5.3-Flash (unknown)","org":"Z.ai (Zhipu AI)","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":52.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":15.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":51.62,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1607.09,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-08-20","release_source":"epoch","price_in":0.06,"price_out":0.2,"cheapest_provider":"clf_ai_gateway","cheapest_model":"glm-5.3-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4-mar-2023","name":"GPT-4 (Mar 2023)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":121.58,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":35.732,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":5.364,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":0.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2023-03-14","release_source":"epoch","price_in":30.0,"price_out":60.0,"cheapest_provider":"openai","cheapest_model":"gpt-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-1-codex","name":"GPT-5.1 Codex","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":400000,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1244.92,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":66,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":57.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1336.28,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2025-11-12","release_source":"epoch","price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"global/gpt-5.1-codex","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-1-codex-max","name":"GPT-5.1-Codex-Max","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":400000,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1208.83,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":223.715,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":60.449,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2025-11-19","release_source":"epoch","price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-5.1-codex-max","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4-mini-low","name":"GPT-5.4 mini (low)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.11,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":36.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":24.561,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-03-17","release_source":"epoch","price_in":0.75,"price_out":4.5,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-astra-none","name":"GPT-6 Astra (none)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":59.583332,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":19.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":82.927,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":53.472,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-09-03","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"azure","cheapest_model":"gpt-6-astra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-sol-unknown-thinking","name":"GPT-6 Sol (unknown thinking)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":36.876,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":83,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":73.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":14427.848,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":null,"release_source":null,"price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-sol-xhigh","name":"GPT-6 Sol (XHigh)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":78.05556,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":28,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":55.093,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":19.732,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-09-22","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-1-sol-medium","name":"GPT-6.1 Sol (medium)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":86.666664,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":27.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":50.23,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":53.241,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-09-29","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.1-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-1-sol-xhigh","name":"GPT-6.1 Sol (XHigh)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":91.666664,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":31.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":55.671,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":38.254,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-09-29","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.1-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-7-high","name":"Grok 4.7 (high)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":58.333332,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":18,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":43.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":57.755,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-09-21","release_source":"curated","price_in":2.0,"price_out":6.0,"cheapest_provider":"azure_ai","cheapest_model":"grok-4.7","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-7-low","name":"Grok 4.7 (low)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":16.666666,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":13.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":33.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":54.861,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-09-21","release_source":"curated","price_in":2.0,"price_out":6.0,"cheapest_provider":"azure_ai","cheapest_model":"grok-4.7","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-3","name":"grok-3","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":137.59,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":36.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":37.24,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2025-02-19","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"azure_ai","cheapest_model":"grok-3","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-code-fast-1","name":"grok-code-fast-1","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":587.73,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":25.843,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1165.08,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":35.06,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":null,"release_source":null,"price_in":0.2,"price_out":1.5,"cheapest_provider":"azure_ai","cheapest_model":"grok-code-fast-1","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k3-1-pro-max","name":"Kimi K3.1 Pro (max)","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1932.75,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":93.33,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":46.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":57.61,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-10-23","release_source":"epoch","price_in":0.2403,"price_out":8.4023,"cheapest_provider":"openrouter","cheapest_model":"moonshotai/kimi-k3.1-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k3-1-pro-unknown","name":"Kimi K3.1 Pro (unknown)","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":65.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":24.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":91,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":60.764,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-10-23","release_source":"epoch","price_in":0.2403,"price_out":8.4023,"cheapest_provider":"openrouter","cheapest_model":"moonshotai/kimi-k3.1-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"meta-llama-3-70b-instruct","name":"Meta-Llama-3-70B-Instruct","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":54.16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":40.562,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":4.306,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2024-04-18","release_source":"epoch","price_in":0.12,"price_out":0.3,"cheapest_provider":"hyperbolic","cheapest_model":"meta-llama/Meta-Llama-3-70B-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"meta-llama-3-8b-instruct","name":"Meta-Llama-3-8B-Instruct","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":43.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":26.073,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":40.98,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":1.944,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2024-04-18","release_source":"epoch","price_in":0.03,"price_out":0.06,"cheapest_provider":"deepinfra","cheapest_model":"meta-llama/Meta-Llama-3-8B-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"mimo-v2-pro","name":"mimo-v2-pro","org":"Xiaomi Corp","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":785.17,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":15.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning from everyday-life context","label":"CL-bench Life","trust":2,"value":6.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench_life","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1433.49,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":null,"release_source":null,"price_in":1.1,"price_out":3.3,"cheapest_provider":"aihubmix","cheapest_model":"mimo-v2-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"minimax-m3-none","name":"MiniMax-M3 (none)","org":"MiniMax","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":81.313,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":26.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-06-01","release_source":"epoch","price_in":0.225,"price_out":0.9,"cheapest_provider":"empiriolabs","cheapest_model":"minimax-m3","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-large-2407","name":"mistral-large-2407","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":131072,"open_weight":true,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":61.21,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":49.021,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":8.472,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":22.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2024-07-24","release_source":"epoch","price_in":2.0,"price_out":6.0,"cheapest_provider":"openrouter","cheapest_model":"mistralai/mistral-large-2407","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-large-2411","name":"mistral-large-2411","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":65.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":60.82,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":51.326,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":7.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2024-11-18","release_source":"epoch","price_in":2.0,"price_out":6.0,"cheapest_provider":"vertex_ai","cheapest_model":"mistral-large-2411","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-medium-2505","name":"mistral-medium-2505","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":62.27,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":59.533,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":4.52,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":32.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2025-05-07","release_source":"epoch","price_in":0.4,"price_out":2.0,"cheapest_provider":"azure_ai","cheapest_model":"mistral-medium-2505","price_observed_at":"2026-11-18T15:47Z"},{"id":"nemotron-3-super","name":"Nemotron 3 Super","org":"Nvidia","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":213.9,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":3.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":35.995,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":38.01,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-03-11","release_source":"epoch","price_in":0.075,"price_out":0.4,"cheapest_provider":"openrouter","cheapest_model":"nvidia/nemotron-3-super-120b-a12b","price_observed_at":"2026-11-18T15:47Z"},{"id":"o1-low","name":"o1 (low)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":11,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":8.421,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":74.242,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":53.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2024-12-05","release_source":"curated","price_in":15.0,"price_out":60.0,"cheapest_provider":"azure","cheapest_model":"o1","price_observed_at":"2026-11-18T15:47Z"},{"id":"o1-mini","name":"o1-mini","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0.83,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":70.7,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"Aider official benchmark","label":"Aider Refactoring","trust":4,"value":44.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/refactor.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_refactor","source_label":"Aider Aider Refactoring leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":135.28,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2024-09-12","release_source":"curated","price_in":1.1,"price_out":4.4,"cheapest_provider":"azure","cheapest_model":"o1-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"o3","name":"o3","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":200000,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":81.3,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":147.05,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Multimodal","trust":4,"value":35.98,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multimodal","source_label":"SWE-bench Multimodal leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":58.4,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":4,"released_at":"2025-04-16","release_source":"curated","price_in":2.0,"price_out":8.0,"cheapest_provider":"azure","cheapest_model":"o3","price_observed_at":"2026-11-18T15:47Z"},{"id":"o3-mini","name":"o3-mini","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":200000,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":60.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":140.65,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":4,"value":32.33,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":42.4,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":4,"released_at":"2025-01-31","release_source":"curated","price_in":1.1,"price_out":4.4,"cheapest_provider":"azure","cheapest_model":"o3-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"o4-mini","name":"o4-mini","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":200000,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":72,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":145.22,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Multimodal","trust":4,"value":33.85,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multimodal","source_label":"SWE-bench Multimodal leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":45,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":4,"released_at":"2025-04-16","release_source":"curated","price_in":1.0,"price_out":4.0,"cheapest_provider":"replicate","cheapest_model":"openai/o4-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen-max-2025-01-25","name":"qwen-max-2025-01-25","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":21.8,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":66.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":56.124,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":16.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2025-01-25","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-235b-a22b-instruct-2507","name":"Qwen3-235B-A22B-Instruct-2507","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":78.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":23.52,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":83,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":38.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2025-07-25","release_source":"epoch","price_in":0.09,"price_out":0.55,"cheapest_provider":"deepinfra","cheapest_model":"Qwen/Qwen3-235B-A22B-Instruct-2507","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-30b-a3b-instruct-2507","name":"qwen3-30b-a3b-instruct-2507","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":67.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":55.619,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":22.37,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":62.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2025-07-29","release_source":"epoch","price_in":0.1,"price_out":0.3,"cheapest_provider":"nebius","cheapest_model":"Qwen/Qwen3-30B-A3B-Instruct-2507","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-coder-next","name":"qwen3-coder-next","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":80.52,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":36.227,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":34.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-02-02","release_source":"epoch","price_in":0.12,"price_out":0.8,"cheapest_provider":"openrouter","cheapest_model":"qwen/qwen3-coder-next","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-5-35b-a3b-none","name":"qwen3.5-35b-a3b (none)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":81.187,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":70,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":29.282,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-02-24","release_source":"epoch","price_in":0.057,"price_out":0.459,"cheapest_provider":"empiriolabs","cheapest_model":"qwen3-5-35b-a3b","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-7-flash","name":"Qwen3.7 Flash","org":"Alibaba","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":19.298,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":82.323,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":15,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":86.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-07-27","release_source":"epoch","price_in":0.03,"price_out":0.13,"cheapest_provider":"empiriolabs","cheapest_model":"qwen3-7-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-8-27b-xhigh","name":"Qwen3.8-27B (XHigh)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":42.36111,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":5.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":46.644,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":45,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-08-14","release_source":"epoch","price_in":0.06,"price_out":2.2,"cheapest_provider":"openrouter","cheapest_model":"qwen/qwen3.8-27b","price_observed_at":"2026-11-18T15:47Z"},{"id":"ring-2-6-1t","name":"Ring-2.6-1T","org":"inclusionAI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":432.57,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":3.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":88.27,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":45.023,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":"2026-05-14","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"trinity-large-thinking","name":"Trinity Large Thinking","org":"Arcee AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":40.625,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":15.625,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1236.61,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":4,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"beam-1-0-unknown","name":"Beam 1.0 (unknown)","org":"Reflection","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":13.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":18,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":48.495,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-10-05","release_source":"epoch","price_in":0.4,"price_out":1.2,"cheapest_provider":"baseten","cheapest_model":"reflection/Beam-1.0","price_observed_at":"2026-11-18T15:47Z"},{"id":"c4ai-command-r-08-2024","name":"c4ai-command-r-08-2024","org":"CohereLabs","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":46.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":26.767677,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":9.19,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2024-08-30","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-3-7-sonnet-16k-thinking","name":"Claude 3.7 Sonnet (16k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":76.768,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":56.092,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":46.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2025-02-24","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"replicate","cheapest_model":"anthropic/claude-3.7-sonnet","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-4-sonnet","name":"Claude 4 Sonnet","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":4,"value":56.67,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Multimodal","trust":2,"value":35.59,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_multimodal","source_label":"SWE-bench Multimodal leaderboard (unverified)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":70.8,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":3,"released_at":"2025-05-22","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"replicate","cheapest_model":"anthropic/claude-4-sonnet","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-4-5-opus","name":"Claude 4.5 Opus","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Multilingual","trust":4,"value":70.7,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multilingual","source_label":"SWE-bench Multilingual leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Test","trust":2,"value":52.62,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_test","source_label":"SWE-bench Test leaderboard (unverified)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":74.4,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":3,"released_at":"2025-11-24","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-haiku-4-5","name":"Claude Haiku 4.5","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":200000,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.25,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":142.76,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"tool use across 1,000 MCP tasks","label":"MCP Atlas","trust":4,"value":40.2,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/mcp_atlas","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"mcp_atlas","source_label":"Scale Labs MCP Atlas leaderboard","higher_is_better":true}],"n_cells":3,"released_at":"2025-10-15","release_source":"curated","price_in":1.0,"price_out":5.0,"cheapest_provider":"anthropic","cheapest_model":"claude-haiku-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-5-5-unknown","name":"Claude Sonnet 5.5 (unknown)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":39.262,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":75.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":11165.278,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-09-28","release_source":"curated","price_in":2.0,"price_out":10.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-2-0","name":"claude-2.0","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":51.85,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":34.659,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":2.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2023-07-11","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-sonnet-4-20250514-32k","name":"claude-sonnet-4-20250514_32K","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":655.35,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":78.283,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":71.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2025-05-22","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-r1-0528","name":"DeepSeek R1 (0528)","org":"deepseek-ai","color":"#64748b","category":"local","params_b":null,"context_window":163840,"open_weight":true,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":71.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":78.78788,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":85,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":3,"released_at":null,"release_source":null,"price_in":0.4,"price_out":0.4,"cheapest_provider":"hyperbolic","cheapest_model":"deepseek-ai/DeepSeek-R1","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v3-0324","name":"DeepSeek V3 0324","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":55.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":81.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":42,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":3,"released_at":"2025-03-24","release_source":"curated","price_in":0.2,"price_out":0.6,"cheapest_provider":"lambda_ai","cheapest_model":"deepseek-v3-0324","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-flash-0731-high","name":"DeepSeek V4 Flash 0731 (high)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":55.97222,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1580.2,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":57.05,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-07-31","release_source":"epoch","price_in":0.06,"price_out":0.18,"cheapest_provider":"deepinfra","cheapest_model":"deepseek-ai/DeepSeek-V4-Flash-0731","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-flash-0731-unknown","name":"DeepSeek V4 Flash 0731 (unknown)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":18.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":56,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":61.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-07-31","release_source":"epoch","price_in":0.06,"price_out":0.18,"cheapest_provider":"deepinfra","cheapest_model":"deepseek-ai/DeepSeek-V4-Flash-0731","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-pro-0813-none","name":"DeepSeek V4 Pro 0813 (None)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0.8333333,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":39.931,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-08-13","release_source":"epoch","price_in":0.111,"price_out":4.44,"cheapest_provider":"openrouter","cheapest_model":"deepseek/deepseek-v4-pro-0813","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-1-flash-none","name":"DeepSeek V4.1 Flash (none)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.25,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":35.532,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-09-09","release_source":"epoch","price_in":0.01,"price_out":1.2,"cheapest_provider":"coralbricks","cheapest_model":"deepseek-v4.1-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-1-flash-1017-high","name":"DeepSeek V4.1 Flash 1017 (High)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":68.611115,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":45.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":9.101,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-10-17","release_source":"epoch","price_in":0.036,"price_out":0.5,"cheapest_provider":"openrouter","cheapest_model":"deepseek/deepseek-v4.1-flash-1017","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-1-flash-1017-none","name":"DeepSeek V4.1 Flash 1017 (None)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":2.0833333,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":38.542,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-10-17","release_source":"epoch","price_in":0.036,"price_out":0.5,"cheapest_provider":"openrouter","cheapest_model":"deepseek/deepseek-v4.1-flash-1017","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-r1-distill-qwen-32b","name":"DeepSeek-R1-Distill-Qwen-32B","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":19.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":64.141,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":55.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2025-01-20","release_source":"epoch","price_in":0.15,"price_out":0.15,"cheapest_provider":"nscale","cheapest_model":"deepseek-ai/DeepSeek-R1-Distill-Qwen-32B","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v3-1-thinking","name":"DeepSeek-V3.1 (thinking)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":82.67,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":24.29,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":38.36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":null,"release_source":null,"price_in":0.25,"price_out":0.95,"cheapest_provider":"deepinfra","cheapest_model":"deepseek-ai/DeepSeek-V3.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-0-pro-exp-feb-2025","name":"Gemini 2.0 Pro Exp (Feb 2025)","org":"Google DeepMind","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":0.69,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":41.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":65.657,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2025-02-05","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-5-flash-preview-apr-2025","name":"Gemini 2.5 Flash Preview (Apr 2025)","org":"Google DeepMind","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":47.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":12.08,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":73.056,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2025-04-17","release_source":"epoch","price_in":0.3,"price_out":2.5,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-1-0-pro-001","name":"gemini-1.0-pro-001","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":45.86,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":33.965,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":1.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2023-12-13","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-1-5-flash-8b-001","name":"gemini-1.5-flash-8b-001","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":50.04,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":32.955,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":4.583,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2024-10-03","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-0-flash-exp","name":"gemini-2.0-flash-exp","org":"Google DeepMind,Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":69.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":22.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":18.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-3-5-flash-medium","name":"gemini-3.5-flash-medium","org":"Google","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1463,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1476,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1284,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":3,"released_at":"2026-05-19","release_source":"epoch","price_in":1.5,"price_out":9.0,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-3.5-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-exp-1206","name":"gemini-exp-1206","org":"Google DeepMind,Google","color":"#64748b","category":"preview","params_b":null,"context_window":1048576,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":80.5,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":38.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":31.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":null,"release_source":null,"price_in":0.3,"price_out":2.5,"cheapest_provider":"gemini","cheapest_model":"gemini-exp-1206","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemma-2-27b","name":"gemma-2-27b","org":"Google DeepMind","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":48,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":118.61,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":7.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2024-06-24","release_source":"epoch","price_in":0.65,"price_out":0.65,"cheapest_provider":"openrouter","cheapest_model":"google/gemma-2-27b-it","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-4-7-novita","name":"GLM-4.7 (Novita)","org":"Z.ai (Zhipu AI)","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":15.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"learning from everyday-life context","label":"CL-bench Life","trust":2,"value":10.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench_life","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":47.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2025-12-22","release_source":"epoch","price_in":0.4,"price_out":1.75,"cheapest_provider":"deepinfra","cheapest_model":"zai-org/GLM-4.7","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-5-3-low","name":"GLM-5.3 (low)","org":"Z.ai (Zhipu AI)","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":14.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":33.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":42.014,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-08-14","release_source":"epoch","price_in":0.0699,"price_out":1.4,"cheapest_provider":"openrouter","cheapest_model":"z-ai/glm-5.3","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-5-3-flash-high","name":"GLM-5.3-Flash (High)","org":"Z.ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":303.55,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":50.13889,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":31.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-08-20","release_source":"epoch","price_in":0.06,"price_out":0.2,"cheapest_provider":"clf_ai_gateway","cheapest_model":"glm-5.3-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-3-5","name":"GPT-3.5","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":4,"value":0.33,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Test","trust":4,"value":0.17,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_test","source_label":"SWE-bench Test leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":0.4,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":3,"released_at":"2022-11-30","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-4-1106","name":"GPT-4 (1106)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":4,"value":18,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Test","trust":4,"value":12.47,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_test","source_label":"SWE-bench Test leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":22.4,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":3,"released_at":"2023-11-06","release_source":"name","price_in":30.0,"price_out":60.0,"cheapest_provider":"openai","cheapest_model":"gpt-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4o-mini","name":"GPT-4o mini","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":128000,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":55.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":123.75,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2024-07-18","release_source":"curated","price_in":0.15,"price_out":0.6,"cheapest_provider":"azure","cheapest_model":"global-standard/gpt-4o-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4o-2024-08-06","name":"gpt-4o-2024-08-06","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":128000,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":71.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":23.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"Aider official benchmark","label":"Aider Refactoring","trust":4,"value":49.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/refactor.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_refactor","source_label":"Aider Aider Refactoring leaderboard","higher_is_better":true}],"n_cells":3,"released_at":"2024-08-06","release_source":"name","price_in":2.5,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"global-standard/gpt-4o-2024-08-06","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5","name":"GPT-5","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":272000,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":88,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":150,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":71.8,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":3,"released_at":"2025-08-07","release_source":"curated","price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-mini","name":"GPT-5 mini","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":272000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":146.01,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Multilingual","trust":4,"value":39.7,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multilingual","source_label":"SWE-bench Multilingual leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":59.8,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":3,"released_at":"2025-08-07","release_source":"curated","price_in":0.25,"price_out":2.0,"cheapest_provider":"azure","cheapest_model":"gpt-5-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-mini-low","name":"GPT-5 mini (low)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0.83,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":18.246,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2025-08-07","release_source":"curated","price_in":0.25,"price_out":2.0,"cheapest_provider":"azure","cheapest_model":"gpt-5-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-1","name":"GPT-5.1","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":272000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":150.97,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1403,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":66,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":3,"released_at":"2025-11-12","release_source":"curated","price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"global/gpt-5.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-3-codex-xhigh","name":"GPT-5.3 Codex (xhigh)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1655.22,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":16.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":77.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-02-05","release_source":"epoch","price_in":1.75,"price_out":14.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.3-codex","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4","name":"GPT-5.4","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":1050000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":157.87,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1471,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1279,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true}],"n_cells":3,"released_at":"2026-03-05","release_source":"epoch","price_in":2.5,"price_out":15.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-high","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-luna-high","name":"GPT-6 Luna (High)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":31.38889,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":15.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":50.347,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-09-22","release_source":"epoch","price_in":0.1,"price_out":0.5,"cheapest_provider":"azure","cheapest_model":"gpt-6-luna","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-luna-low","name":"GPT-6 Luna (Low)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":4.5833335,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":2.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":46.875,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-09-22","release_source":"epoch","price_in":0.1,"price_out":0.5,"cheapest_provider":"azure","cheapest_model":"gpt-6-luna","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-luna-medium","name":"GPT-6 Luna (Medium)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":18.055555,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":10.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":50.926,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-09-22","release_source":"epoch","price_in":0.1,"price_out":0.5,"cheapest_provider":"azure","cheapest_model":"gpt-6-luna","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-luna-none","name":"GPT-6 Luna (None)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":1.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":43.056,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-09-22","release_source":"epoch","price_in":0.1,"price_out":0.5,"cheapest_provider":"azure","cheapest_model":"gpt-6-luna","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-sol-high","name":"GPT-6 Sol (High)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":68.888885,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":25.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":54.861,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-09-22","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-sol-low","name":"GPT-6 Sol (Low)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":31.527779,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":16.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":50.231,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-09-22","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-sol-medium","name":"GPT-6 Sol (Medium)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":57.77778,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":24.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":53.819,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-09-22","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-sol-none","name":"GPT-6 Sol (None)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.6666666,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":47.338,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-09-22","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-1-sol-high","name":"GPT-6.1 Sol (High)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":91.666664,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":30,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":55.787,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-09-29","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.1-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-1-sol-low","name":"GPT-6.1 Sol (Low)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":76.666664,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":24.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":53.241,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-09-29","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.1-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-1-terra-unknown-thinking","name":"GPT-6.1 Terra (unknown thinking)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":42.082,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":72.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":9107.458,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":null,"release_source":null,"price_in":1.0,"price_out":5.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.1-terra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-1-terra-xhigh","name":"GPT-6.1 Terra (xhigh)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":26.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":53.356,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":28.086,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-10-26","release_source":"epoch","price_in":1.0,"price_out":5.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.1-terra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-oss-20b-low","name":"gpt-oss-20b (low)","org":"OpenAI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":53.157,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":40.278,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2025-08-05","release_source":"epoch","price_in":0.0145,"price_out":0.07,"cheapest_provider":"darkbloom","cheapest_model":"gpt-oss-20b","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-oss-20b-medium","name":"gpt-oss-20b (medium)","org":"OpenAI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":60.795,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":65.278,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":36.83,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2025-08-05","release_source":"epoch","price_in":0.0145,"price_out":0.07,"cheapest_provider":"darkbloom","cheapest_model":"gpt-oss-20b","price_observed_at":"2026-11-18T15:47Z"},{"id":"granite-4-0-350m","name":"granite-4.0-350m","org":"IBM","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":11.237,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":0.278,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2025-10-28","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"granite-4-0-micro","name":"granite-4.0-micro","org":"IBM","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":28.283,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":2.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2025-10-02","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"granite-4-1-3b","name":"granite-4.1-3b","org":"ibm-granite","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":31.7,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":86.88,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":49.83,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":3,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"granite-4-1-8b","name":"granite-4.1-8b","org":"ibm-granite","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":41.96,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":92.49,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":55.99,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":3,"released_at":null,"release_source":null,"price_in":0.05,"price_out":0.1,"cheapest_provider":"wandb","cheapest_model":"ibm-granite/granite-4.1-8b","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-3-mini-beta-high","name":"grok-3-mini-beta (high)","org":"xAI","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":75.505,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":77.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":42.58,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"grok-4-20-beta-0309-reasoning","name":"grok-4.20-beta-0309-reasoning","org":"xAI","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1416,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1472,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1255,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true}],"n_cells":3,"released_at":null,"release_source":null,"price_in":1.25,"price_out":2.5,"cheapest_provider":"xai","cheapest_model":"grok-4.20-beta-0309-reasoning","price_observed_at":"2026-11-18T15:47Z"},{"id":"inkling-2-small-xhigh","name":"Inkling 2 Small (XHigh)","org":"Thinking Machines","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":81.666664,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":25.07,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":43.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-11-05","release_source":"epoch","price_in":0.3,"price_out":1.0,"cheapest_provider":"nebius","cheapest_model":"thinkingmachines/Inkling-2-Small","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k2-thinking-turbo","name":"Kimi K2 Thinking Turbo","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":84.217,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":83.056,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1337,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"laguna-m-1","name":"Laguna M.1","org":"Poolside","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":15.625,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1347.49,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-04-28","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"lfm2-8b-a1b","name":"LFM2-8B-A1B","org":"LiquidAI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":29.29,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":84.38,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":37.42,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":3,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"llama-3-1-405b","name":"Llama 3.1-405B","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":127.87,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":96.8,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":61.6,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":3,"released_at":"2024-07-23","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"llama-4-maverick-fp8","name":"Llama 4 Maverick (FP8)","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":172.97,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":66.982,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":20.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2025-04-05","release_source":"curated","price_in":0.2,"price_out":0.6,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"meta/llama-4-maverick","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-3-2-1b-instruct","name":"Llama-3.2-1B-Instruct","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":6.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":18.686869,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":0.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2024-09-24","release_source":"epoch","price_in":0.02,"price_out":0.02,"cheapest_provider":"novita","cheapest_model":"meta-llama/llama-3.2-1b-instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"mellum2-12b-a2-5b-base","name":"Mellum2-12B-A2.5B-Base","org":"JetBrains","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":31.31,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":81.73,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":59.31,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":3,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mellum2-12b-a2-5b-base-pretrain","name":"Mellum2-12B-A2.5B-Base-Pretrain","org":"JetBrains","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":31.31,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":81.73,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":59.31,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":3,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mercury-2-5-unknown","name":"Mercury 2.5 (unknown)","org":"Inception Labs","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":38.542,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-09-08","release_source":"epoch","price_in":0.04,"price_out":0.15,"cheapest_provider":"openrouter","cheapest_model":"inception/mercury-2.5","price_observed_at":"2026-11-08T04:47Z"},{"id":"ministral-3b-2410","name":"ministral-3b-2410","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":51.73,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":25.253,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":5.48,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2024-10-16","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mistral-small-3-2","name":"Mistral Small 3.2","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":59.87,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":26.389,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2025-06-20","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mistral-small-4","name":"Mistral Small 4","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":497.62,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":70.93,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":20.61,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-03-16","release_source":"epoch","price_in":0.15,"price_out":0.6,"cheapest_provider":"empiriolabs","cheapest_model":"mistral-small-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-large-2402","name":"mistral-large-2402","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":55.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":38.763,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":1.944,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2024-02-26","release_source":"epoch","price_in":8.0,"price_out":24.0,"cheapest_provider":"azure","cheapest_model":"mistral-large-2402","price_observed_at":"2026-11-18T15:47Z"},{"id":"mixtral-8x22b-instruct-v0-1","name":"Mixtral-8x22B-Instruct-v0.1","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":7.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":55.14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":3.17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2024-04-17","release_source":"epoch","price_in":0.6,"price_out":0.6,"cheapest_provider":"nscale","cheapest_model":"mistralai/mixtral-8x22b-instruct-v0.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark-1-1-high","name":"Muse Spark 1.1 (high)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":94.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":49.91,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"geometry optimisation with a solver","label":"Surface Evolver","trust":2,"value":52.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"surface_evolver","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-07-09","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"empiriolabs","cheapest_model":"muse-spark-1-1","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark-1-1-xhigh","name":"Muse Spark 1.1 (xhigh)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":34,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":15.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":58.796,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-07-09","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"empiriolabs","cheapest_model":"muse-spark-1-1","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-wonder-xhigh","name":"Muse Wonder (xhigh)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":48.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"post-training a smaller model","label":"PostTrainBench","trust":2,"value":24.19,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"posttrainbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":15.138,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-10-12","release_source":"epoch","price_in":4.0,"price_out":20.0,"cheapest_provider":"openrouter","cheapest_model":"meta/muse-wonder","price_observed_at":"2026-11-18T15:47Z"},{"id":"nova-2-0-pro-preview-none","name":"Nova 2.0 Pro Preview (none)","org":"Amazon","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":28.125,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"phi-4-mini-instruct","name":"Phi-4-mini-instruct","org":"microsoft","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":52.8,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":10.764,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2025-02-27","release_source":"epoch","price_in":0.075,"price_out":0.3,"cheapest_provider":"azure_ai","cheapest_model":"Phi-4-mini-instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"prokope-7b","name":"ProKope-7B","org":"ProKope-AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":36.87,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":81.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":44.03,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":3,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-coder-32b-instruct","name":"Qwen2.5 Coder 32B Instruct","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":32768,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":71.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":16.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":38,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":3,"released_at":"2024-11-12","release_source":"curated","price_in":0.06,"price_out":0.2,"cheapest_provider":"nscale","cheapest_model":"Qwen/Qwen2.5-Coder-32B-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-4b-instruct-2507","name":"qwen3-4b-instruct-2507","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":62,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":69.6,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":52.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2025-08-06","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-8-27b-low","name":"Qwen3.8-27B (Low)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":22.777779,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":40.046,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-08-14","release_source":"epoch","price_in":0.06,"price_out":2.2,"cheapest_provider":"openrouter","cheapest_model":"qwen/qwen3.8-27b","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-8-27b-medium","name":"Qwen3.8-27B (Medium)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":13.194445,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":39.005,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-08-14","release_source":"epoch","price_in":0.06,"price_out":2.2,"cheapest_provider":"openrouter","cheapest_model":"qwen/qwen3.8-27b","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-8-max","name":"qwen3.8-max","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1672,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1482,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1301,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":3,"released_at":"2026-08-02","release_source":"epoch","price_in":1.65,"price_out":4.951,"cheapest_provider":"deepinfra","cheapest_model":"Qwen/Qwen3.8-Max","price_observed_at":"2026-11-18T15:47Z"},{"id":"seed-oss-36b-instruct","name":"seed-oss-36b-instruct","org":"ByteDance-Seed","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":71.528,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":82.7,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":67.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2025-08-20","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"step-3-7-flash","name":"Step 3.7 Flash","org":"StepFun","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":694.12,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":2.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":40.046,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":"2026-05-29","release_source":"epoch","price_in":0.2,"price_out":1.15,"cheapest_provider":"deepinfra","cheapest_model":"stepfun-ai/Step-3.7-Flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"step-5-preview-unknown","name":"Step 5 Preview (unknown)","org":"StepFun","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":20.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":42,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":58.912,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":3,"released_at":null,"release_source":null,"price_in":1.0,"price_out":2.7,"cheapest_provider":"empiriolabs","cheapest_model":"step-5-preview","price_observed_at":"2026-11-18T15:47Z"},{"id":"swe-llama-13b","name":"SWE-Llama 13B","org":"","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":4,"value":1,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Test","trust":4,"value":0.7,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_test","source_label":"SWE-bench Test leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":1.2,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":3,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"swe-llama-7b","name":"SWE-Llama 7B","org":"Meta","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":4,"value":1.33,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Test","trust":4,"value":0.7,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_test","source_label":"SWE-bench Test leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":1.4,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":3,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"a-x-k1","name":"A.X-K1","org":"skt","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":74,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":81.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"amazon-nova-lite-v1-0","name":"amazon.nova-lite-v1:0","org":"Amazon","color":"#64748b","category":"frontier","params_b":null,"context_window":300000,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":236.25,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":3.64,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2024-12-03","release_source":"epoch","price_in":0.06,"price_out":0.24,"cheapest_provider":"bedrock","cheapest_model":"amazon.nova-lite-v1:0","price_observed_at":"2026-11-18T15:47Z"},{"id":"amazon-nova-pro-v1-0","name":"amazon.nova-pro-v1:0","org":"Amazon","color":"#64748b","category":"frontier","params_b":null,"context_window":300000,"open_weight":false,"cells":[{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":4.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"work tasks in a simulated software company","label":"TheAgentCompany","trust":2,"value":1.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"agent_company","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2024-12-03","release_source":"epoch","price_in":0.8,"price_out":3.2,"cheapest_provider":"bedrock","cheapest_model":"amazon.nova-pro-v1:0","price_observed_at":"2026-11-18T15:47Z"},{"id":"apertus-v1-5-70b","name":"Apertus-v1.5-70B","org":"swiss-ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":84.988625,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":56.38298,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"apertus-v1-5-8b","name":"Apertus-v1.5-8B","org":"swiss-ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":78.46854,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":48.753323,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"beam-1-0-max","name":"Beam 1.0 (max)","org":"Reflection","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1097.53,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":48.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-10-05","release_source":"epoch","price_in":0.4,"price_out":1.2,"cheapest_provider":"baseten","cheapest_model":"reflection/Beam-1.0","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-5-haiku","name":"Claude 3.5 Haiku","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":124.75,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":40.6,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":2,"released_at":"2024-10-22","release_source":"curated","price_in":0.8,"price_out":4.0,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"anthropic/claude-3.5-haiku","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-7-sonnet-32k-thinking","name":"Claude 3.7 Sonnet (32k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":76.768,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":53.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-02-24","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"replicate","cheapest_model":"anthropic/claude-3.7-sonnet","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-7-sonnet-64k-thinking","name":"Claude 3.7 Sonnet (64k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":79.735,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":57.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-02-24","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"replicate","cheapest_model":"anthropic/claude-3.7-sonnet","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-4-5-haiku","name":"Claude 4.5 Haiku","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Multilingual","trust":4,"value":64.7,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multilingual","source_label":"SWE-bench Multilingual leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":66.6,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":2,"released_at":"2025-10-15","release_source":"curated","price_in":1.0,"price_out":5.0,"cheapest_provider":"replicate","cheapest_model":"anthropic/claude-4.5-haiku","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-4-5-sonnet","name":"Claude 4.5 Sonnet","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Multilingual","trust":4,"value":67,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multilingual","source_label":"SWE-bench Multilingual leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":70.6,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2025-09-29","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"replicate","cheapest_model":"anthropic/claude-4.5-sonnet","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-4-6-opus","name":"Claude 4.6 Opus","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Multilingual","trust":4,"value":72,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multilingual","source_label":"SWE-bench Multilingual leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":75.6,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":2,"released_at":"2026-02-05","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-fable-5-1","name":"Claude Fable 5.1","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":1000000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":167.09,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":4,"value":51.3,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/hle-diamond","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"hle_diamond","source_label":"Scale Labs HLE leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2026-09-01","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5-1","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-haiku-5-5-none","name":"Claude Haiku 5.5 (none)","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":75.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":39.26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-10-07","release_source":"epoch","price_in":0.1,"price_out":0.5,"cheapest_provider":"anthropic","cheapest_model":"claude-haiku-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-mythos-preview-early","name":"Claude Mythos Preview (Early)","org":"Anthropic","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"writing working exploits","label":"ExploitBench","trust":2,"value":73.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"exploitbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":1044.78,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":10.0,"price_out":50.0,"cheapest_provider":"anthropic","cheapest_model":"claude-mythos-preview","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-5-128k-thinking","name":"Claude Opus 4.5 (128k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"evidence-based reasoning","label":"EBR-Bench","trust":2,"value":14.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ebr_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":59.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-11-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-5-64k-thinking","name":"Claude Opus 4.5 (64k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":43,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"planning a physical assembly","label":"Furniture Assembly","trust":2,"value":28.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"furniture_assembly","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-11-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-6-32k-thinking","name":"Claude Opus 4.6 (32k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":90.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":93.056,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-02-05","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-6-64k-thinking","name":"Claude Opus 4.6 (64k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":88.763,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":94.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-02-05","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-6-low","name":"Claude Opus 4.6 (low)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":51.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-02-05","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-5-5","name":"Claude Opus 5.5","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":1000000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":168.7,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":4,"value":55,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/hle-diamond","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"hle_diamond","source_label":"Scale Labs HLE leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2026-09-22","release_source":"curated","price_in":4.0,"price_out":20.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-5","name":"Claude Sonnet 4.5","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":1000000,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":3.75,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":147.59,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-09-29","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-6-32k-thinking","name":"Claude Sonnet 4.6 (32k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":87.374,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":85.794,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-02-17","release_source":"epoch","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-6-low","name":"Claude Sonnet 4.6 (low)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":50.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-02-17","release_source":"epoch","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-5-haiku-20241022","name":"claude-3-5-haiku-20241022","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":75.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":28,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2024-10-22","release_source":"name","price_in":1.0,"price_out":5.0,"cheapest_provider":"vertex_ai","cheapest_model":"claude-3-5-haiku@20241022","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-5-sonnet-20240620","name":"claude-3.5-sonnet-20240620","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":77.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"Aider official benchmark","label":"Aider Refactoring","trust":4,"value":64,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/refactor.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_refactor","source_label":"Aider Aider Refactoring leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2024-06-20","release_source":"name","price_in":3.0,"price_out":15.0,"cheapest_provider":"vertex_ai","cheapest_model":"claude-3-5-sonnet@20240620","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-1-20250805-27k","name":"claude-opus-4-1-20250805_27K","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":76.768,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":68.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-08-05","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-opus-4-1-20250805-32k","name":"claude-opus-4-1-20250805_32K","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":12.632,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":2.439,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-08-05","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-opus-4-5-20251101","name":"claude-opus-4-5-20251101","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":200000,"open_weight":false,"cells":[{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1462,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1470,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true}],"n_cells":2,"released_at":"2025-11-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-5-20251101","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-20250514-59k","name":"claude-sonnet-4-20250514_59K","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":79.188,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":68.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-05-14","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"codeact-v2-1-claude-3-5-sonnet-20241022","name":"CodeAct v2.1 (claude-3-5-sonnet-20241022)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":4,"value":41.67,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Test","trust":4,"value":29.38,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_test","source_label":"SWE-bench Test leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"cogito-v2-1-671b","name":"Cogito v2.1 (671B)","org":"Deep Cogito","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":40.972,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-11-19","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"command-a-plus","name":"Command A+","org":"Cohere,Cohere Labs (formerly Cohere for AI)","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":38.542,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-05-20","release_source":"epoch","price_in":0.3,"price_out":1.5,"cheapest_provider":"openrouter","cheapest_model":"cohere/command-a-plus","price_observed_at":"2026-11-18T15:47Z"},{"id":"composer-2-5","name":"Composer 2.5","org":"Cursor","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":27.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":25.64,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-05-18","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-chat-v2-5","name":"DeepSeek Chat V2.5","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":17.8,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"Aider official benchmark","label":"Aider Refactoring","trust":4,"value":31.5,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/refactor.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_refactor","source_label":"Aider Aider Refactoring leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2024-09-05","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-coder-v2-0724","name":"DeepSeek Coder V2 0724","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":72.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"Aider official benchmark","label":"Aider Refactoring","trust":4,"value":32.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/refactor.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_refactor","source_label":"Aider Aider Refactoring leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":157,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":90.9,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2026-09-09","release_source":"epoch","price_in":0.01,"price_out":1.2,"cheapest_provider":"coralbricks","cheapest_model":"deepseek-v4.1-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-1-flash-1017-low","name":"DeepSeek V4.1 Flash 1017 (Low)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":73.611115,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":44.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-10-17","release_source":"epoch","price_in":0.036,"price_out":0.5,"cheapest_provider":"openrouter","cheapest_model":"deepseek/deepseek-v4.1-flash-1017","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-coder-v2-lite-base","name":"DeepSeek-Coder-V2-Lite-Base","org":"deepseek-ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":99.31,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":34.37,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2024-06-13","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-llm-67b-chat","name":"deepseek-llm-67b-chat","org":"DeepSeek","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":24.621,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":0.833,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2023-11-29","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-r1-0528-qwen3-8b","name":"deepseek-r1-0528-qwen3-8b","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":9.28,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":43.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-05-28","release_source":"epoch","price_in":0.06,"price_out":0.09,"cheapest_provider":"novita","cheapest_model":"deepseek/deepseek-r1-0528-qwen3-8b","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-r1-distill-llama-70b","name":"DeepSeek-R1-Distill-Llama-70B","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":55.745,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":51.389,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-01-20","release_source":"epoch","price_in":0.2,"price_out":0.6,"cheapest_provider":"deepinfra","cheapest_model":"deepseek-ai/DeepSeek-R1-Distill-Llama-70B","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-r1-distill-qwen-1-5b","name":"DeepSeek-R1-Distill-Qwen-1.5B","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":33.586,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":21.389,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-01-20","release_source":"epoch","price_in":0.09,"price_out":0.09,"cheapest_provider":"nscale","cheapest_model":"deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-r1-distill-qwen-14b","name":"DeepSeek-R1-Distill-Qwen-14B","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":44.697,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":50.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-01-20","release_source":"epoch","price_in":0.07,"price_out":0.07,"cheapest_provider":"nscale","cheapest_model":"deepseek-ai/DeepSeek-R1-Distill-Qwen-14B","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v2-5","name":"DeepSeek-V2.5","org":"deepseek-ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":72.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":65.83,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2024-09-05","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-v3-2-thinking","name":"DeepSeek-V3.2 (Thinking)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":83.424,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":87.817,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":0.26,"price_out":0.38,"cheapest_provider":"deepinfra","cheapest_model":"deepseek-ai/DeepSeek-V3.2","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v3-2-speciale","name":"DeepSeek-V3.2-Speciale","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":52.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":46.73,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-12-01","release_source":"epoch","price_in":0.28,"price_out":0.4,"cheapest_provider":"gmi","cheapest_model":"deepseek-ai/DeepSeek-V3.2-Speciale","price_observed_at":"2026-11-18T15:47Z"},{"id":"devstral-small-2","name":"Devstral Small 2","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":28.819,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-12-09","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"diffusiongemma-26b-a4b-it","name":"diffusiongemma-26B-A4B-it","org":"google","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":73.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":77.6,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ernie-5-1","name":"ernie-5.1","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1227,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1468,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":0.5634,"price_out":2.5353,"cheapest_provider":"aihubmix","cheapest_model":"ernie-5.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"exaone-4-5-33b","name":"EXAONE-4.5-33B","org":"LGAI-EXAONE","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":80.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":83.3,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-0-flash","name":"Gemini 2.0 Flash","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.3,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":13.52,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2024-12-11","release_source":"curated","price_in":0.15,"price_out":0.6,"cheapest_provider":"vertex_ai","cheapest_model":"gemini-2.0-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-pro-preview-mar-2025","name":"Gemini 2.5 Pro Preview (Mar 2025)","org":"Google DeepMind","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":66.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":51.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-03-31","release_source":"epoch","price_in":1.25,"price_out":10.0,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-pro-preview-low","name":"Gemini 3 Pro Preview (low)","org":"Google DeepMind","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":988.23,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":46.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":2.0,"price_out":12.0,"cheapest_provider":"replicate","cheapest_model":"google/gemini-3-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-8-flash","name":"Gemini 3.8 Flash","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":1048576,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":158.83,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":4,"value":34.3,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/hle-diamond","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"hle_diamond","source_label":"Scale Labs HLE leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2026-09-02","release_source":"epoch","price_in":0.75,"price_out":3.75,"cheapest_provider":"gemini","cheapest_model":"gemini-3.8-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-4-carbon-low","name":"Gemini 4 Carbon (Low)","org":"Google","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":49.583332,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":58.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-10-28","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"gemini","cheapest_model":"gemini-4-carbon","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-4-carbon-medium","name":"Gemini 4 Carbon (Medium)","org":"Google","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":94.166664,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":60.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-10-28","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"gemini","cheapest_model":"gemini-4-carbon","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-4-hydrogen-high","name":"Gemini 4 Hydrogen (high)","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":54.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"hard multi-hour software engineering","label":"FrontierSWE","trust":2,"value":53.113,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontierswe","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-11-12","release_source":"epoch","price_in":1.5,"price_out":7.5,"cheapest_provider":"gemini","cheapest_model":"gemini-4-hydrogen","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-4-hydrogen-unknown","name":"Gemini 4 Hydrogen (unknown)","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":78,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"running a vending business for a year (final money, $)","label":"Vending-Bench 2","trust":2,"value":14549.49,"score_max":25000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"vending_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-11-12","release_source":"epoch","price_in":1.5,"price_out":7.5,"cheapest_provider":"gemini","cheapest_model":"gemini-4-hydrogen","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-4-hydrogen-xhigh","name":"Gemini 4 Hydrogen (xhigh)","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1825.67,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":57.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-11-12","release_source":"epoch","price_in":1.5,"price_out":7.5,"cheapest_provider":"gemini","cheapest_model":"gemini-4-hydrogen","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-flash-preview-09-2025","name":"gemini-2.5-flash-preview-09-2025","org":"Google DeepMind","color":"#64748b","category":"preview","params_b":null,"context_window":1048576,"open_weight":false,"cells":[{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":17.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"work tasks in a simulated software company","label":"TheAgentCompany","trust":2,"value":41.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"agent_company","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":0.3,"price_out":2.5,"cheapest_provider":"vertex_ai","cheapest_model":"gemini-2.5-flash-preview-09-2025","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemma-2-9b","name":"Gemma 2 9B","org":"Google DeepMind","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":112.28,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":45.1,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2024-06-24","release_source":"epoch","price_in":0.2,"price_out":0.2,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"google/gemma-2-9b","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemma-2-27b-it","name":"gemma-2-27b-it","org":"Google DeepMind","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":36.49,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":1.389,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2024-06-24","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemma-2-9b-it","name":"gemma-2-9b-it","org":"Google DeepMind","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":27.462,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":0.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2024-06-24","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemma-3-1b-it","name":"gemma-3-1b-it","org":"Google DeepMind","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":19.949,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":1.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-03-12","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemma-4-31b","name":"gemma-4-31b","org":"Google","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1425,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1262,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true}],"n_cells":2,"released_at":"2026-04-02","release_source":"openrouter","price_in":0.1642,"price_out":0.4689,"cheapest_provider":"kyma","cheapest_model":"gemma-4-31b","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-4-7-flash","name":"glm-4.7-flash","org":"zai-org","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":75.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":58.333,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-01-19","release_source":"openrouter","price_in":0.024,"price_out":0.16,"cheapest_provider":"clf_ai_gateway","cheapest_model":"glm-4.7-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-4-7-flash-none","name":"glm-4.7-flash (none)","org":"Z.ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":45.139,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-01-19","release_source":"openrouter","price_in":0.024,"price_out":0.16,"cheapest_provider":"clf_ai_gateway","cheapest_model":"glm-4.7-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-5-2-max","name":"glm-5.2-max","org":"Z.ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1605,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-07","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-06)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1476,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"glm-5-3-high","name":"GLM-5.3 (high)","org":"Z.ai (Zhipu AI)","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1317.4,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":38,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-08-14","release_source":"epoch","price_in":0.0699,"price_out":1.4,"cheapest_provider":"openrouter","cheapest_model":"z-ai/glm-5.3","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-5-3-flash-low","name":"GLM-5.3-Flash (Low)","org":"Z.ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":27.916666,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":26.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-08-20","release_source":"epoch","price_in":0.06,"price_out":0.2,"cheapest_provider":"clf_ai_gateway","cheapest_model":"glm-5.3-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-3-5-turbo-nov-2023","name":"GPT-3.5 Turbo (Nov 2023)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":112.57,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":28.03,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2023-11-06","release_source":"epoch","price_in":0.5,"price_out":1.5,"cheapest_provider":"azure","cheapest_model":"gpt-3.5-turbo","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4-0613","name":"GPT-4 (0613)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":8192,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":67.7,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":2,"value":23.67,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard (unverified)","higher_is_better":true}],"n_cells":2,"released_at":"2023-06-13","release_source":"name","price_in":30.0,"price_out":60.0,"cheapest_provider":"openai","cheapest_model":"gpt-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4-turbo-preview-january-2024","name":"GPT-4 Turbo Preview (January 2024)","org":"OpenAI","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":42.266,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":5.366,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2024-01-25","release_source":"epoch","price_in":10.0,"price_out":30.0,"cheapest_provider":"openai","cheapest_model":"gpt-4-turbo","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4-turbo-preview-nov-2023","name":"GPT-4 Turbo Preview (Nov 2023)","org":"OpenAI","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":42.361,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":4.045,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2023-11-06","release_source":"epoch","price_in":10.0,"price_out":30.0,"cheapest_provider":"openai","cheapest_model":"gpt-4-turbo","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4-0125-preview","name":"gpt-4-0125-preview","org":"OpenAI","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":66.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"Aider official benchmark","label":"Aider Refactoring","trust":4,"value":33.7,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/refactor.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_refactor","source_label":"Aider Aider Refactoring leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":10.0,"price_out":30.0,"cheapest_provider":"azure","cheapest_model":"gpt-4-0125-preview","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4-1106-preview","name":"gpt-4-1106-preview","org":"OpenAI","color":"#64748b","category":"preview","params_b":null,"context_window":128000,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":65.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"Aider official benchmark","label":"Aider Refactoring","trust":4,"value":50.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/refactor.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_refactor","source_label":"Aider Aider Refactoring leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":10.0,"price_out":30.0,"cheapest_provider":"azure","cheapest_model":"gpt-4-1106-preview","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4-turbo","name":"gpt-4-turbo","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":128000,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":61.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":9.78,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2023-11-06","release_source":"curated","price_in":10.0,"price_out":30.0,"cheapest_provider":"openai","cheapest_model":"gpt-4-turbo","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4-5","name":"GPT-4.5","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0.8,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":136.5,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-02-27","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-4o-2024-11-20","name":"gpt-4o-2024-11-20","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":128000,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":71.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":18.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2024-11-20","release_source":"name","price_in":2.5,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"global-standard/gpt-4o-2024-11-20","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-nano","name":"GPT-5 nano","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":272000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":139.09,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":34.8,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2025-08-07","release_source":"curated","price_in":0.05,"price_out":0.4,"cheapest_provider":"azure","cheapest_model":"gpt-5-nano","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-codex","name":"GPT-5-codex","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":272000,"open_weight":false,"cells":[{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":44.326,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":54.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-09-15","release_source":"epoch","price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-5-codex","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-1-codex-mini","name":"GPT-5.1 Codex Mini","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":400000,"open_weight":false,"cells":[{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":61.573,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1247,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-11-12","release_source":"epoch","price_in":0.25,"price_out":2.0,"cheapest_provider":"azure","cheapest_model":"global/gpt-5.1-codex-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-2-chat-latest-20260210","name":"gpt-5.2-chat-latest-20260210","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1476,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1278,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true}],"n_cells":2,"released_at":"2026-02-10","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-5-4-mini-medium","name":"GPT-5.4 mini (medium)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":4.44,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-03-17","release_source":"epoch","price_in":0.75,"price_out":4.5,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4-nano-medium","name":"GPT-5.4 nano (medium)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.94,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-03-17","release_source":"epoch","price_in":0.2,"price_out":1.25,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-nano","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-astra","name":"GPT-6 Astra","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":922000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":169.2,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":4,"value":60.6,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/hle-diamond","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"hle_diamond","source_label":"Scale Labs HLE leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2026-09-03","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"azure","cheapest_model":"gpt-6-astra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-luna-unknown-thinking","name":"GPT-6 Luna (unknown thinking)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":31.236,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":64,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":0.1,"price_out":0.5,"cheapest_provider":"azure","cheapest_model":"gpt-6-luna","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-sol","name":"GPT-6 Sol","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":922000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":163.82,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":4,"value":33.8,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/hle-diamond","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"hle_diamond","source_label":"Scale Labs HLE leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2026-09-22","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-1-terra-high","name":"GPT-6.1 Terra (high)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":25.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":51.62,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-10-26","release_source":"epoch","price_in":1.0,"price_out":5.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.1-terra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-1-terra-low","name":"GPT-6.1 Terra (low)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":15.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":52.431,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-10-26","release_source":"epoch","price_in":1.0,"price_out":5.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.1-terra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-1-terra-medium","name":"GPT-6.1 Terra (medium)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":21.429,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":51.62,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-10-26","release_source":"epoch","price_in":1.0,"price_out":5.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.1-terra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-1-terra-none","name":"GPT-6.1 Terra (none)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":4.857,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":47.801,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-10-26","release_source":"epoch","price_in":1.0,"price_out":5.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.1-terra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-oss-120b-low","name":"gpt-oss-120b (low)","org":"OpenAI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":35.995,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-08-05","release_source":"curated","price_in":0.03,"price_out":0.17,"cheapest_provider":"wandb","cheapest_model":"openai/gpt-oss-120b","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-oss-120b-medium","name":"gpt-oss-120b (medium)","org":"OpenAI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":41.88,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-08-05","release_source":"curated","price_in":0.03,"price_out":0.17,"cheapest_provider":"wandb","cheapest_model":"openai/gpt-oss-120b","price_observed_at":"2026-11-18T15:47Z"},{"id":"granite-4-0-1b","name":"granite-4.0-1b","org":"IBM","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":23.99,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":0.833,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-10-28","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"grok-3-mini","name":"Grok 3 mini","org":"xAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":139.95,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":42.58,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-02-19","release_source":"curated","price_in":0.25,"price_out":1.27,"cheapest_provider":"azure_ai","cheapest_model":"grok-3-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4","name":"Grok 4","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":79.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":146.96,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-07-09","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"azure_ai","cheapest_model":"grok-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-7","name":"Grok 4.7","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":156.43,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":4,"value":23.4,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/hle-diamond","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"hle_diamond","source_label":"Scale Labs HLE leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2026-09-21","release_source":"curated","price_in":2.0,"price_out":6.0,"cheapest_provider":"azure_ai","cheapest_model":"grok-4.7","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-7-medium","name":"Grok 4.7 (medium)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":58.75,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":41.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-09-21","release_source":"curated","price_in":2.0,"price_out":6.0,"cheapest_provider":"azure_ai","cheapest_model":"grok-4.7","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-3-mini-beta-low","name":"grok-3-mini-beta (low)","org":"xAI","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":76.263,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":62.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"grok-4-1","name":"grok-4-1","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":39,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1210.95,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-11-17","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"grok-4-20-beta1","name":"grok-4.20-beta1","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1189,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1475,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"grok-4-20-multi-agent-beta-0309","name":"grok-4.20-multi-agent-beta-0309","org":"xAI","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1204,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1471,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":1.25,"price_out":2.5,"cheapest_provider":"xai","cheapest_model":"grok-4.20-multi-agent-beta-0309","price_observed_at":"2026-11-18T15:47Z"},{"id":"hermes-2-theta-llama-3-70b","name":"Hermes-2-Theta-Llama-3-70B","org":"Nous Research,Arcee AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":37.468,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":2.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2024-06-20","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"hy4-preview","name":"Hy4-preview","org":"tencent","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":92.3,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1632,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":0.417,"price_out":1.2505,"cheapest_provider":"gmi","cheapest_model":"tencent/hy4","price_observed_at":"2026-11-18T15:47Z"},{"id":"inkling-2-small-unknown","name":"Inkling 2 Small (unknown)","org":"Thinking Machines","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":46.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":52.315,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-11-05","release_source":"epoch","price_in":0.3,"price_out":1.0,"cheapest_provider":"nebius","cheapest_model":"thinkingmachines/Inkling-2-Small","price_observed_at":"2026-11-18T15:47Z"},{"id":"joyai-llm-flash","name":"JoyAI-LLM-Flash","org":"jdopensource","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":74.43,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":81.02,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"k-exaone-236b-a23b","name":"K-EXAONE-236B-A23B","org":"LGAI-EXAONE","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":79.1,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":83.8,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"kimi-k2","name":"Kimi K2","org":"Moonshot AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":59.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":65.4,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2025-07-11","release_source":"curated","price_in":0.55,"price_out":2.2,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"moonshotai/kimi-k2","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k2-5-fireworks","name":"Kimi K2.5 (Fireworks)","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":87.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":92.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-01-27","release_source":"epoch","price_in":0.45,"price_out":2.25,"cheapest_provider":"deepinfra","cheapest_model":"moonshotai/Kimi-K2.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k3","name":"Kimi K3","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":158.6,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":93.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2026-07-16","release_source":"epoch","price_in":0.192,"price_out":16.0,"cheapest_provider":"openrouter","cheapest_model":"moonshotai/kimi-k3","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k2-instruct-0905","name":"Kimi-K2-Instruct-0905","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":66.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":36.68,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-09-05","release_source":"epoch","price_in":0.5,"price_out":2.0,"cheapest_provider":"deepinfra","cheapest_model":"moonshotai/Kimi-K2-Instruct-0905","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k2-5-thinking","name":"kimi-k2.5-thinking","org":"Moonshot AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1430,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true},{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1252,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":0.45,"price_out":2.25,"cheapest_provider":"deepinfra","cheapest_model":"moonshotai/Kimi-K2.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k3-1-pro","name":"Kimi-K3.1-Pro","org":"moonshotai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":161.36,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":94.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2026-10-23","release_source":"epoch","price_in":0.2403,"price_out":8.4023,"cheapest_provider":"openrouter","cheapest_model":"moonshotai/kimi-k3.1-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"laguna-xs-2","name":"Laguna XS.2","org":"Poolside","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1302.4,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-04-28","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"llada2-0-flash","name":"LLaDA2.0-flash","org":"inclusionAI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":96.06,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":73.36,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"llama-3-1-70b","name":"Llama 3.1-70B","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":124.72,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":52.47,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2024-07-23","release_source":"epoch","price_in":0.6,"price_out":0.6,"cheapest_provider":"cerebras","cheapest_model":"llama-3.1-70b","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-3-1-8b","name":"Llama 3.1-8B","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":110.78,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":36.6,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2024-07-23","release_source":"epoch","price_in":0.03,"price_out":0.05,"cheapest_provider":"llamagate","cheapest_model":"llama-3.1-8b","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-4-scout","name":"Llama 4 Scout","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":127.77,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-04-05","release_source":"curated","price_in":0.1,"price_out":0.3,"cheapest_provider":"openrouter","cheapest_model":"meta-llama/llama-4-scout","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-2-70b-chat-hf","name":"Llama-2-70b-chat-hf","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":26.326,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2023-07-18","release_source":"epoch","price_in":1.0,"price_out":1.0,"cheapest_provider":"anyscale","cheapest_model":"meta-llama/Llama-2-70b-chat-hf","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-3-1-tulu-3-70b-dpo","name":"Llama-3.1-Tulu-3-70B-DPO","org":"Allen Institute for AI,University of Washington","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":46.275,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":4.444,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2024-11-21","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"llama-3-2-11b-vision-instruct","name":"Llama-3.2-11B-Vision-Instruct","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":16.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2024-09-24","release_source":"epoch","price_in":0.049,"price_out":0.049,"cheapest_provider":"deepinfra","cheapest_model":"meta-llama/Llama-3.2-11B-Vision-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-4-maverick-17b-128e-instruct","name":"Llama-4-Maverick-17B-128E-Instruct","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":46.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-04-06","release_source":"epoch","price_in":0.63,"price_out":1.8,"cheapest_provider":"sambanova","cheapest_model":"Llama-4-Maverick-17B-128E-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"magistral-medium-1-2","name":"Magistral Medium 1.2","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":39.236,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-09-18","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"magistral-small-2506","name":"magistral-small-2506","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":56.061,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":30,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-06-10","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mcts-refine-7b","name":"MCTS Refine 7B","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":2,"value":16.33,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard (unverified)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":23.2,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"minicpm5-1b","name":"MiniCPM5-1B","org":"openbmb","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":26.26,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":48.85,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ministral-8b-2410","name":"ministral-8b-2410","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":45.69,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":27.146,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2024-10-16","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mistral-7b-instruct-v0-3","name":"Mistral-7B-Instruct-v0.3","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":15.183,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":0.278,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2024-05-27","release_source":"epoch","price_in":0.1,"price_out":0.1,"cheapest_provider":"ovhcloud","cheapest_model":"Mistral-7B-Instruct-v0.3","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-nemo-instruct-2407","name":"Mistral-Nemo-Instruct-2407","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":17.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":48.61,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2024-07-18","release_source":"epoch","price_in":0.029,"price_out":0.03,"cheapest_provider":"deepinfra","cheapest_model":"mistralai/Mistral-Nemo-Instruct-2407","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-small-2501","name":"mistral-small-2501","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":45.297,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":5.278,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-01-25","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mistral-small-3-2501","name":"mistral-small-3-2501","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":47.285,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":6.667,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-01-30","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mistral-small-3-1-2503","name":"mistral-small-3.1-2503","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":41.919,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":3.889,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-03-17","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mistral-small-3-2-2506","name":"mistral-small-3.2-2506","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":49.053,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":30.278,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-06-20","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"muse-ember-unknown","name":"Muse Ember (unknown)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":64.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":72.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-11-09","release_source":"epoch","price_in":0.1,"price_out":0.5,"cheapest_provider":"openrouter","cheapest_model":"meta/muse-ember","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-glimmer-high","name":"Muse Glimmer (high)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":2.571,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":44.907,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-08-10","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"muse-spark-1-3","name":"Muse Spark 1.3","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":158.25,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":4,"value":25.4,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/hle-diamond","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"hle_diamond","source_label":"Scale Labs HLE leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2026-09-02","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"empiriolabs","cheapest_model":"muse-spark-1-3","price_observed_at":"2026-11-18T15:47Z"},{"id":"nova-2-0-pro-preview-low","name":"Nova 2.0 Pro Preview (low)","org":"Amazon","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":38.657,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"nova-2-0-pro-preview-medium","name":"Nova 2.0 Pro Preview (medium)","org":"Amazon","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":42.708,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"nvidia-nemotron-3-super-120b-a12b-bf16","name":"NVIDIA-Nemotron-3-Super-120B-A12B-BF16","org":"nvidia","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":79.23,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":83.73,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"nvidia-nemotron-3-ultra-550b-a55b-bf16","name":"NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16","org":"nvidia","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":87,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":86.8,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"nvidia-nemotron-3-ultra-550b-a55b-nvfp4","name":"NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4","org":"nvidia","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":87.9,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":86.8,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"nvidia-nemotron-3-5-lightning-30b-a3b-bf16","name":"NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16","org":"nvidia","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":75.44,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":81.94,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":0.15,"price_out":1.0,"cheapest_provider":"gmi","cheapest_model":"nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16","price_observed_at":"2026-11-18T15:47Z"},{"id":"nvidia-nemotron-3-5-lightning-30b-a3b-nvfp4","name":"NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4","org":"nvidia","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":75.57,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":81.62,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"nvidia-nemotron-nano-9b-v2","name":"nvidia.nemotron-nano-9b-v2","org":"NVIDIA","color":"#64748b","category":"flash","params_b":null,"context_window":128000,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":25,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-08-18","release_source":"epoch","price_in":0.04,"price_out":0.16,"cheapest_provider":"deepinfra","cheapest_model":"nvidia/NVIDIA-Nemotron-Nano-9B-v2","price_observed_at":"2026-11-18T15:47Z"},{"id":"o1","name":"o1","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":200000,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":84.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":141.18,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2024-12-05","release_source":"curated","price_in":15.0,"price_out":60.0,"cheapest_provider":"azure","cheapest_model":"o1","price_observed_at":"2026-11-18T15:47Z"},{"id":"o1-unknown-thinking","name":"o1 (unknown thinking)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":5.65,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level questions across domains","label":"Humanity's Last Exam","trust":2,"value":7.96,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"hle","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":15.0,"price_out":60.0,"cheapest_provider":"azure","cheapest_model":"o1","price_observed_at":"2026-11-18T15:47Z"},{"id":"o1-mini-high","name":"o1-mini (high)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":62.374,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":46.944,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2024-09-12","release_source":"curated","price_in":1.1,"price_out":4.4,"cheapest_provider":"azure","cheapest_model":"o1-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"o1-mini-2024-09-12","name":"o1-mini-2024-09-12","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":27.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":32.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2024-09-12","release_source":"name","price_in":1.1,"price_out":4.4,"cheapest_provider":"azure","cheapest_model":"o1-mini-2024-09-12","price_observed_at":"2026-11-18T15:47Z"},{"id":"open-mistral-7b","name":"open-mistral-7b","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":42.54,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":13.226,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2023-09-27","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"open-mixtral-8x7b","name":"open-mixtral-8x7b","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":49.61,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":29.83,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2023-12-11","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"phi-3-medium-128k-instruct","name":"Phi-3-medium-128k-instruct","org":"Microsoft","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":27.588,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":51.91,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2024-04-23","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"phi-3-medium-4k-instruct","name":"Phi-3-medium-4k-instruct","org":"microsoft","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":91,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":55.7,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"phi-3-mini-4k-instruct","name":"Phi-3-mini-4k-instruct","org":"Microsoft","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":85.7,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":45.66,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2024-04-23","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"phi-3-5-mini-instruct","name":"Phi-3.5-mini-instruct","org":"microsoft","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":86.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":47.87,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen-plus-2025-01-25","name":"qwen-plus-2025-01-25","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":48.106,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":17.778,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-01-25","release_source":"name","price_in":0.4,"price_out":1.2,"cheapest_provider":"dashscope","cheapest_model":"qwen-plus-2025-01-25","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen-plus-2025-04-28","name":"qwen-plus-2025-04-28","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":81.07,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"language-model capability assessment","label":"LMCA","trust":2,"value":23.96,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"lmca","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-04-28","release_source":"name","price_in":0.4,"price_out":1.2,"cheapest_provider":"dashscope","cheapest_model":"qwen-plus-2025-04-28","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen-turbo-2024-11-01","name":"qwen-turbo-2024-11-01","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":41.793,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":6.111,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2024-11-01","release_source":"name","price_in":0.05,"price_out":0.2,"cheapest_provider":"dashscope","cheapest_model":"qwen-turbo-2024-11-01","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen2-72b","name":"Qwen2-72B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":122.33,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":89.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2024-06-07","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-7b","name":"Qwen2-7B","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":79.9,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":40.73,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-7b-plus-72b","name":"Qwen2.5 (7B + 72B)","org":"Qwen","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":4,"value":24.67,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":32.8,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-32b-instruct","name":"qwen2.5-32b-instruct","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":46.086,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":7.361,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2024-09-17","release_source":"epoch","price_in":0.06,"price_out":0.2,"cheapest_provider":"nebius","cheapest_model":"Qwen/Qwen2.5-32B-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen2-5-72b","name":"Qwen2.5-72B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":127.21,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":71.59,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2024-09-19","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-omni-7b","name":"Qwen2.5-Omni-7B","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":88.7,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":47,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-vl-72b-instruct","name":"Qwen2.5-VL-72B-Instruct","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":64.94348,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true},{"note":"computer-use tasks (v1)","label":"OSWorld","trust":2,"value":5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"os_world","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2024-09-19","release_source":"epoch","price_in":0.25,"price_out":0.75,"cheapest_provider":"nebius","cheapest_model":"Qwen/Qwen2.5-VL-72B-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-1-7b-none","name":"qwen3-1.7b (none)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":30.619,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":8.056,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-04-29","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-14b-none","name":"qwen3-14b (none)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":53.409,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":25.833,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-04-28","release_source":"epoch","price_in":0.08,"price_out":0.24,"cheapest_provider":"nebius","cheapest_model":"Qwen/Qwen3-14B","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-30b-a3b-none","name":"Qwen3-30B-A3B (none)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":50.379,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":25.556,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-04-28","release_source":"epoch","price_in":0.08,"price_out":0.25,"cheapest_provider":"gmi","cheapest_model":"Qwen/Qwen3-30B-A3B","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-32b-none","name":"Qwen3-32B (none)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":54.104,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":23.056,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-04-29","release_source":"curated","price_in":0.08,"price_out":0.23,"cheapest_provider":"ovhcloud","cheapest_model":"Qwen3-32B","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-8b-none","name":"qwen3-8b (none)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":47.475,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":22.222,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-04-28","release_source":"epoch","price_in":0.04,"price_out":0.14,"cheapest_provider":"llamagate","cheapest_model":"qwen3-8b","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-coder-30b-a3b-instruct","name":"Qwen3-Coder-30B-A3B-Instruct","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":2,"value":49.67,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard (unverified)","higher_is_better":true},{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":60.4,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":2,"released_at":"2025-07-31","release_source":"curated","price_in":0.07,"price_out":0.27,"cheapest_provider":"novita","cheapest_model":"qwen/qwen3-coder-30b-a3b-instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-5-122b-a10b","name":"Qwen3.5-122B-A10B","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":86.6,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":86.7,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2026-02-24","release_source":"epoch","price_in":0.1126,"price_out":0.9008,"cheapest_provider":"aihubmix","cheapest_model":"qwen3.5-122b-a10b","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-5-4b-none","name":"qwen3.5-4b (none)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"competition maths (AIME 2024-25 style)","label":"OTIS Mock AIME","trust":2,"value":55.833,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"aime","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-02-24","release_source":"epoch","price_in":0.04,"price_out":0.07,"cheapest_provider":"empiriolabs","cheapest_model":"qwen3-5-4b","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-8-27b-no-thinking","name":"Qwen3.8 27B (no thinking)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0.286,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":36.227,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":0.06,"price_out":2.2,"cheapest_provider":"openrouter","cheapest_model":"qwen/qwen3.8-27b","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-8-max-0902","name":"Qwen3.8 Max (0902)","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":155.98,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1672,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":1.65,"price_out":4.951,"cheapest_provider":"deepinfra","cheapest_model":"Qwen/Qwen3.8-Max","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-8-max-0902-unknown","name":"Qwen3.8 Max (0902) (unknown)","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":17.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":52.083,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":1.65,"price_out":4.951,"cheapest_provider":"deepinfra","cheapest_model":"Qwen/Qwen3.8-Max","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":91.7,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1632,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"reka-flash-3","name":"Reka Flash 3","org":"rekaai","color":"#64748b","category":"local","params_b":null,"context_window":65536,"open_weight":true,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":29.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2025-03-10","release_source":"epoch","price_in":0.1,"price_out":0.2,"cheapest_provider":"openrouter","cheapest_model":"rekaai/reka-flash-3","price_observed_at":"2026-11-18T15:47Z"},{"id":"solar-pro-3","name":"Solar Pro 3","org":"Upstage","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"scientific research code","label":"SciCode","trust":2,"value":25.463,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"scicode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2026-01-27","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"solar-open2-250b","name":"Solar-Open2-250B","org":"upstage","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":86.3,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":86.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"step-3-5-flash","name":"Step-3.5-Flash","org":"stepfun-ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":83.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":84.4,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2026-01-29","release_source":"openrouter","price_in":0.1,"price_out":0.3,"cheapest_provider":"empiriolabs","cheapest_model":"step-3-5-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"step-5-preview-high","name":"step-5-preview-high","org":"StepFun","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1570,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true},{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1570.15,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":1.0,"price_out":2.7,"cheapest_provider":"empiriolabs","cheapest_model":"step-5-preview","price_observed_at":"2026-11-18T15:47Z"},{"id":"tencent-hy4-preview-unknown","name":"Tencent Hy4 preview (unknown)","org":"Tencent","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":35,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":75,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"wizardlm-2-8x22b","name":"WizardLM-2-8x22B","org":"Microsoft","color":"#64748b","category":"local","params_b":null,"context_window":65535,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":44.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":43.434,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":2,"released_at":"2024-04-15","release_source":"epoch","price_in":0.48,"price_out":0.48,"cheapest_provider":"deepinfra","cheapest_model":"microsoft/WizardLM-2-8x22B","price_observed_at":"2026-11-18T15:47Z"},{"id":"yi-6b","name":"Yi 6B","org":"01.AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":94.06,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":26.51,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2023-11-22","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"yi-1-5-34b-chat","name":"Yi-1.5-34B-Chat","org":"01.AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":31.976,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true},{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":52.29,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":2,"released_at":"2024-05-13","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"yi-lightning","name":"yi-lightning","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":65.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true},{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":12.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":2,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"a-x-k2","name":"A.X-K2","org":"skt","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":85.6,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ai-co-mathematician","name":"AI co-mathematician","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"research-level maths, tier 4","label":"FrontierMath T4","trust":2,"value":75.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t4","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-05-08","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ai21-jamba-large-1-5","name":"AI21-Jamba-Large-1.5","org":"ai21labs","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":49.46,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"amazon-nova-premier-v1-0","name":"Amazon.nova Premier v1:0","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":1000000,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":42.4,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":2.5,"price_out":12.5,"cheapest_provider":"bedrock","cheapest_model":"amazon.nova-premier-v1:0","price_observed_at":"2026-11-18T15:47Z"},{"id":"amazon-nova-micro-v1-0","name":"amazon.nova-micro-v1:0","org":"Amazon","color":"#64748b","category":"frontier","params_b":null,"context_window":128000,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-12-03","release_source":"epoch","price_in":0.035,"price_out":0.14,"cheapest_provider":"bedrock","cheapest_model":"amazon.nova-micro-v1:0","price_observed_at":"2026-11-18T15:47Z"},{"id":"antigravity-cli-plus-gemini-3-1-pro","name":"Antigravity CLI + Gemini 3.1 Pro","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":2.73,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"antigravity-cli-plus-gemini-4-argon","name":"Antigravity CLI + Gemini 4 Argon","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":55.45,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"antigravity-cli-plus-gemini-4-carbon","name":"Antigravity CLI + Gemini 4 Carbon","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":64.55,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"antigravity-cli-plus-gemini-4-hydrogen","name":"Antigravity CLI + Gemini 4 Hydrogen","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":56.36,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"api-gpt-4o-search","name":"api-gpt-4o-search","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1006,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"aya-expanse-32b","name":"aya-expanse-32b","org":"CohereLabs","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":45.41,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"aya-expanse-8b","name":"aya-expanse-8b","org":"CohereLabs","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":33.74,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"baichuan-2-7b","name":"Baichuan 2-7B","org":"Baichuan","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":83.25,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-09-20","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"baichuan2-13b","name":"Baichuan2-13B","org":"Baichuan","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":91.59,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-09-06","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"beam-1-0","name":"Beam 1.0","org":"Reflection","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":150.87,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-05","release_source":"epoch","price_in":0.4,"price_out":1.2,"cheapest_provider":"baseten","cheapest_model":"reflection/Beam-1.0","price_observed_at":"2026-11-18T15:47Z"},{"id":"beam-1-0-high","name":"Beam 1.0 (High)","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":20,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-05","release_source":"epoch","price_in":0.4,"price_out":1.2,"cheapest_provider":"baseten","cheapest_model":"reflection/Beam-1.0","price_observed_at":"2026-11-18T15:47Z"},{"id":"beam-1-0-low","name":"Beam 1.0 (Low)","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":2.7777777,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-05","release_source":"epoch","price_in":0.4,"price_out":1.2,"cheapest_provider":"baseten","cheapest_model":"reflection/Beam-1.0","price_observed_at":"2026-11-18T15:47Z"},{"id":"beam-1-0-medium","name":"Beam 1.0 (Medium)","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":6.388889,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-05","release_source":"epoch","price_in":0.4,"price_out":1.2,"cheapest_provider":"baseten","cheapest_model":"reflection/Beam-1.0","price_observed_at":"2026-11-18T15:47Z"},{"id":"beam-1-0-xhigh","name":"Beam 1.0 (XHigh)","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":23.472221,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-05","release_source":"epoch","price_in":0.4,"price_out":1.2,"cheapest_provider":"baseten","cheapest_model":"reflection/Beam-1.0","price_observed_at":"2026-11-18T15:47Z"},{"id":"beam-1-0-501b-a23b","name":"Beam-1.0-501B-A23B","org":"reflection","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":90.6,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-16","release_source":"openrouter","price_in":0.3,"price_out":1.0,"cheapest_provider":"openrouter","cheapest_model":"reflection/beam-1.0-501b-a23b","price_observed_at":"2026-11-18T15:47Z"},{"id":"c4ai-command-r-v01","name":"c4ai-command-r-v01","org":"CohereLabs","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":37.9,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"c4ai-command-r7b-12-2024","name":"c4ai-command-r7b-12-2024","org":"CohereLabs","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":26.767677,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"cerebras-gpt-13b","name":"Cerebras-GPT-13B","org":"Cerebras Systems","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":48.59,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-03-20","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"chatglm2-6b","name":"chatglm2-6b","org":"Z.ai","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":73.4,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-06-24","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"chatgpt-4o-01-29-2025","name":"chatgpt-4o-01-29-2025","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":66.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-01-29","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"chatgpt-4o-latest-2025-02-15","name":"chatgpt-4o-latest (2025-02-15)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":27.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-02-15","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"chatgpt-4o-latest-2025-03-29","name":"chatgpt-4o-latest (2025-03-29)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":45.3,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-03-29","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-3-haiku","name":"Claude 3 Haiku","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":117.79,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-03-13","release_source":"curated","price_in":0.25,"price_out":1.25,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"anthropic/claude-3-haiku","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-sonnet","name":"Claude 3 Sonnet","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":119.65,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-03-04","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"vertex_ai","cheapest_model":"claude-3-sonnet","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-5-sonnet-october-2024","name":"Claude 3.5 Sonnet (October 2024)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":133.34,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-10-22","release_source":"epoch","price_in":3.0,"price_out":15.0,"cheapest_provider":"snowflake","cheapest_model":"claude-3-5-sonnet","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-7","name":"Claude 3.7","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-3-7-16k","name":"Claude 3.7 (16K)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0.7,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-3-7-1k","name":"Claude 3.7 (1K)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0.4,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-3-7-8k","name":"Claude 3.7 (8K)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0.9,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-3-7-sonnet-12k-thinking","name":"Claude 3.7 Sonnet (12k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":46.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-02-24","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"replicate","cheapest_model":"anthropic/claude-3.7-sonnet","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-7-sonnet-2k-thinking","name":"Claude 3.7 Sonnet (2k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":43.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-02-24","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"replicate","cheapest_model":"anthropic/claude-3.7-sonnet","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-7-sonnet-8k-thinking","name":"Claude 3.7 Sonnet (8k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":83.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-02-24","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"replicate","cheapest_model":"anthropic/claude-3.7-sonnet","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-3-7-sonnet-w-review-heavy","name":"Claude 3.7 Sonnet w/ Review Heavy","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":62.4,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-4-opus","name":"Claude 4 Opus","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":67.6,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-05-22","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"snowflake","cheapest_model":"claude-4-opus","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-4-7-high","name":"Claude 4.7 (High)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":68.33,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-4-7-low","name":"Claude 4.7 (Low)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":62.08,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-4-7-max","name":"Claude 4.7 (Max)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":75.83,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-4-7-medium","name":"Claude 4.7 (Medium)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":67.5,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-code-plus-deepseek-v4-pro-0813","name":"Claude Code + DeepSeek V4 Pro 0813","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":6.67,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-code-plus-deepseek-v4-1-flash-1017","name":"Claude Code + DeepSeek V4.1 Flash 1017","org":"","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":38.48,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-code-plus-fable-5","name":"Claude Code + Fable 5","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":44.85,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-code-plus-fable-5-1","name":"Claude Code + Fable 5.1","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":57.58,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-code-plus-fable-5-2","name":"Claude Code + Fable 5.2","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":65.76,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-code-plus-glm-5-3","name":"Claude Code + GLM-5.3","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":41.21,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-code-plus-glm-5-3-flash","name":"Claude Code + GLM-5.3-Flash","org":"","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":36.97,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-code-plus-haiku-5-5","name":"Claude Code + Haiku 5.5","org":"","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":40,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-code-plus-hy4","name":"Claude Code + Hy4","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":37.88,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-code-plus-hy4-preview","name":"Claude Code + Hy4 Preview","org":"","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":19.09,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-code-plus-kimi-k3","name":"Claude Code + Kimi K3","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":23.03,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-code-plus-kimi-k3-1-pro","name":"Claude Code + Kimi K3.1 Pro","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":59.7,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-code-plus-mistral-large-4","name":"Claude Code + Mistral Large 4","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":27.58,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-code-plus-opus-4-8","name":"Claude Code + Opus 4.8","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":23.64,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-code-plus-opus-5","name":"Claude Code + Opus 5","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":53.33,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-code-plus-opus-5-5","name":"Claude Code + Opus 5.5","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":65.76,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-code-plus-qwen3-8-max-0902","name":"Claude Code + Qwen3.8-Max-0902","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":26.97,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-code-plus-sonnet-5","name":"Claude Code + Sonnet 5","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":12.12,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-code-plus-sonnet-5-5","name":"Claude Code + Sonnet 5.5","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":63.94,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-fable-5-with-fallback-max","name":"Claude Fable 5 (with fallback, max)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":93.876,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-06-09","release_source":"epoch","price_in":10.0,"price_out":50.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-2-with-fallback-max","name":"Claude Fable 5.2 (with fallback, max)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":96.023,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-11-09","release_source":"epoch","price_in":8.0,"price_out":40.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5-2","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-haiku-4-5-59k-thinking","name":"Claude Haiku 4.5 (59k thinking)","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-10-15","release_source":"curated","price_in":1.0,"price_out":5.0,"cheapest_provider":"anthropic","cheapest_model":"claude-haiku-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-haiku-4-5-low","name":"Claude Haiku 4.5 (low)","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":45.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-10-15","release_source":"curated","price_in":1.0,"price_out":5.0,"cheapest_provider":"anthropic","cheapest_model":"claude-haiku-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-haiku-4-5-thinking-16k","name":"Claude Haiku 4.5 (Thinking 16K)","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":2.78,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-10-15","release_source":"curated","price_in":1.0,"price_out":5.0,"cheapest_provider":"anthropic","cheapest_model":"claude-haiku-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-haiku-4-5-thinking-1k","name":"Claude Haiku 4.5 (Thinking 1K)","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.25,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-10-15","release_source":"curated","price_in":1.0,"price_out":5.0,"cheapest_provider":"anthropic","cheapest_model":"claude-haiku-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-haiku-4-5-thinking-32k","name":"Claude Haiku 4.5 (Thinking 32K)","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":4.03,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-10-15","release_source":"curated","price_in":1.0,"price_out":5.0,"cheapest_provider":"anthropic","cheapest_model":"claude-haiku-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-haiku-4-5-thinking-8k","name":"Claude Haiku 4.5 (Thinking 8K)","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.67,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-10-15","release_source":"curated","price_in":1.0,"price_out":5.0,"cheapest_provider":"anthropic","cheapest_model":"claude-haiku-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-instant","name":"Claude Instant","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":114,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-08-09","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-opus-4-thinking-16k","name":"Claude Opus 4 (Thinking 16K)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":8.61,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-05-22","release_source":"curated","price_in":15.0,"price_out":75.0,"cheapest_provider":"gmi","cheapest_model":"anthropic/claude-opus-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-thinking-1k","name":"Claude Opus 4 (Thinking 1K)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-05-22","release_source":"curated","price_in":15.0,"price_out":75.0,"cheapest_provider":"gmi","cheapest_model":"anthropic/claude-opus-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-thinking-8k","name":"Claude Opus 4 (Thinking 8K)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":4.52,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-05-22","release_source":"curated","price_in":15.0,"price_out":75.0,"cheapest_provider":"gmi","cheapest_model":"anthropic/claude-opus-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-1","name":"Claude Opus 4.1","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":144.77,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-08-05","release_source":"epoch","price_in":15.0,"price_out":75.0,"cheapest_provider":"openrouter","cheapest_model":"anthropic/claude-opus-4.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-5","name":"Claude Opus 4.5","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":200000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":151.88,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-11-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-5-48k-thinking","name":"Claude Opus 4.5 (48k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":22,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-11-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-5-59k-thinking","name":"Claude Opus 4.5 (59k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":14,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-11-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-5-low","name":"Claude Opus 4.5 (low)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":53.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-11-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-6-120k-high","name":"Claude Opus 4.6 (120K, High)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":69.17,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-02-05","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-6-120k-low","name":"Claude Opus 4.6 (120K, Low)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":64.58,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-02-05","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-6-120k-max","name":"Claude Opus 4.6 (120K, Max)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":68.75,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-02-05","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-6-120k-medium","name":"Claude Opus 4.6 (120K, Medium)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":66.25,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-02-05","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-6-medium","name":"Claude Opus 4.6 (medium)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":53.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-02-05","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-6","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-7-low","name":"Claude Opus 4.7 (low)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"chess tactics puzzles","label":"Chess Puzzles","trust":2,"value":26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"chess_puzzles","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-04-16","release_source":"epoch","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-7","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-8-low","name":"Claude Opus 4.8 (Low)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":62.22,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-05-28","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-8","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-8-medium","name":"Claude Opus 4.8 (Medium)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":71.67,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-05-28","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-8","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-5-5-with-fallback-max","name":"Claude Opus 5.5 (with fallback, max)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":94.066,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-09-22","release_source":"curated","price_in":4.0,"price_out":20.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-3-5","name":"Claude Sonnet 3.5","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Multimodal","trust":4,"value":12.19,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_multimodal","source_label":"SWE-bench Multimodal leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2024-06-20","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-sonnet-4-thinking-16k","name":"Claude Sonnet 4 (Thinking 16K)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":5.93,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-05-22","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"gmi","cheapest_model":"anthropic/claude-sonnet-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-thinking-1k","name":"Claude Sonnet 4 (Thinking 1K)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0.85,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-05-22","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"gmi","cheapest_model":"anthropic/claude-sonnet-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-thinking-8k","name":"Claude Sonnet 4 (Thinking 8K)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":2.12,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-05-22","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"gmi","cheapest_model":"anthropic/claude-sonnet-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-5-12k-thinking","name":"Claude Sonnet 4.5 (12k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":54.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-09-29","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-5-2k-thinking","name":"Claude Sonnet 4.5 (2k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":52.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-09-29","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-5-48k-thinking","name":"Claude Sonnet 4.5 (48k thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":17,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-09-29","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-5-low","name":"Claude Sonnet 4.5 (low)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":47.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-09-29","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-5-thinking-16k","name":"Claude Sonnet 4.5 (Thinking 16K)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":6.94,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-09-29","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-5-thinking-1k","name":"Claude Sonnet 4.5 (Thinking 1K)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":5.83,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-09-29","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-5-thinking-32k","name":"Claude Sonnet 4.5 (Thinking 32K)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":13.61,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-09-29","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-5-thinking-8k","name":"Claude Sonnet 4.5 (Thinking 8K)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":6.94,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-09-29","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-5","name":"Claude Sonnet 5","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":1000000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":156.9,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-06-30","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"aihubmix","cheapest_model":"claude-sonnet-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-5-no-thinking","name":"Claude Sonnet 5 (no thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":16,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-06-30","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"aihubmix","cheapest_model":"claude-sonnet-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":1000000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":166.59,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-09-28","release_source":"curated","price_in":2.0,"price_out":10.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-fable-5-2","name":"claude-fable-5-2","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":1000000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":173.5,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-11-09","release_source":"epoch","price_in":8.0,"price_out":40.0,"cheapest_provider":"anthropic","cheapest_model":"claude-fable-5-2","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-haiku-4-5-20251001-16k","name":"claude-haiku-4-5-20251001_16K","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":44.15,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-10-01","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-haiku-4-5-20251001-1k","name":"claude-haiku-4-5-20251001_1K","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":31.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-10-01","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-haiku-5-5","name":"claude-haiku-5-5","org":"Anthropic","color":"#64748b","category":"flash","params_b":null,"context_window":1000000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":160.29,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-07","release_source":"epoch","price_in":0.1,"price_out":0.5,"cheapest_provider":"anthropic","cheapest_model":"claude-haiku-5-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-instant-1-2","name":"claude-instant-1.2","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":45.78,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-08-09","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-opus-4-1-20250805-24k","name":"claude-opus-4-1-20250805_24K","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":21,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-08-05","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-opus-4-1-search","name":"claude-opus-4-1-search","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1148,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-opus-4-20250514","name":"claude-opus-4-20250514","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":72,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-05-22","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-opus-4-5-20251101-high-32k","name":"claude-opus-4-5-20251101-high-32k","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1474,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true}],"n_cells":1,"released_at":"2025-11-24","release_source":"curated","price_in":5.0,"price_out":25.0,"cheapest_provider":"anthropic","cheapest_model":"claude-opus-4-5-20251101","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-opus-4-5-search","name":"claude-opus-4-5-search","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1180,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-opus-4-6-search","name":"claude-opus-4-6-search","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1253,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-opus-4-search","name":"claude-opus-4-search","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1126,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-sonnet-4-20250514-12k","name":"claude-sonnet-4-20250514_12K","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":45.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-05-14","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-sonnet-4-20250514-2k","name":"claude-sonnet-4-20250514_2K","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"research-report quality","label":"DeepResearch Bench","trust":2,"value":46.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"deepresearch","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-05-14","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-sonnet-4-5-thinking","name":"claude-sonnet-4-5 (thinking)","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"tool use across 1,000 MCP tasks","label":"MCP Atlas","trust":4,"value":59.5,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/mcp_atlas","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"mcp_atlas","source_label":"Scale Labs MCP Atlas leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-09-29","release_source":"curated","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-5","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-5-20250929","name":"claude-sonnet-4-5-20250929","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":1000000,"open_weight":false,"cells":[{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1450,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":"2025-09-29","release_source":"name","price_in":3.0,"price_out":15.0,"cheapest_provider":"anthropic","cheapest_model":"claude-sonnet-4-5-20250929","price_observed_at":"2026-11-18T15:47Z"},{"id":"claude-sonnet-4-5-search","name":"claude-sonnet-4-5-search","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1158,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-sonnet-4-6-search","name":"claude-sonnet-4-6-search","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1221,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"claude-sonnet-5-search","name":"claude-sonnet-5-search","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1194,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"codeact-v1-8","name":"CodeAct v1.8","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":4,"value":26.67,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"codeqwen1-5-7b","name":"CodeQwen1.5-7B","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":74.53,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-04-15","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"codeqwen-7b-chat-v1-5-q8-0","name":"codeqwen:7b-chat-v1.5-q8_0","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":34.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"codestral-25-01","name":"Codestral 25.01","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":11.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-01-13","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"codestral-22b-v0-1-q4-k-m","name":"Codestral-22B-v0.1-Q4_K_M","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":48.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"codestral-2405","name":"codestral-2405","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":51.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.2,"price_out":0.6,"cheapest_provider":"vertex_ai","cheapest_model":"codestral@2405","price_observed_at":"2026-11-18T15:47Z"},{"id":"codestral-2508","name":"codestral-2508","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":137.78,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-07-30","release_source":"epoch","price_in":0.3,"price_out":0.9,"cheapest_provider":"mistral","cheapest_model":"codestral-2508","price_observed_at":"2026-11-18T15:47Z"},{"id":"codestral-22b-v0-1-q8-0","name":"codestral:22b-v0.1-q8_0","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":48.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"codex-plus-gpt-5-4-mini","name":"Codex + GPT-5.4 Mini","org":"","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":1.82,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"codex-plus-gpt-5-6-luna","name":"Codex + GPT-5.6 Luna","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":17.58,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"codex-plus-gpt-5-6-sol","name":"Codex + GPT-5.6 Sol","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":38.48,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"codex-plus-gpt-5-6-terra","name":"Codex + GPT-5.6 Terra","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":21.52,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"codex-plus-gpt-6-astra","name":"Codex + GPT-6 Astra","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":60.3,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"codex-plus-gpt-6-luna","name":"Codex + GPT-6 Luna","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":15.15,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"codex-plus-gpt-6-sol","name":"Codex + GPT-6 Sol","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":50.3,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"codex-plus-gpt-6-1-sol","name":"Codex + GPT-6.1 Sol","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":60.91,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"codex-plus-gpt-6-1-terra","name":"Codex + GPT-6.1 Terra","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":53.33,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"codex-plus-gpt-6-2-astra","name":"Codex + GPT-6.2 Astra","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":62.12,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"codex-plus-gpt-6-2-sol","name":"Codex + GPT-6.2 Sol","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":61.52,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"codex-mini-latest","name":"Codex Mini (Latest)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.27,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":1.5,"price_out":6.0,"cheapest_provider":"azure","cheapest_model":"codex-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"command-r-08-24","name":"Command R (08-24)","org":"Cohere","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":38.3,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.15,"price_out":0.6,"cheapest_provider":"openrouter","cheapest_model":"cohere/command-r-08-2024","price_observed_at":"2026-11-18T15:47Z"},{"id":"command-r-plus","name":"Command R+","org":"Cohere,Cohere Labs (formerly Cohere for AI)","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":31.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2024-08-30","release_source":"epoch","price_in":2.5,"price_out":10.0,"cheapest_provider":"openrouter","cheapest_model":"cohere/command-r-plus-08-2024","price_observed_at":"2026-11-18T15:47Z"},{"id":"command-r-plus-08-24","name":"Command R+ (08-24)","org":"Cohere","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":38.3,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":2.5,"price_out":10.0,"cheapest_provider":"openrouter","cheapest_model":"cohere/command-r-plus-08-2024","price_observed_at":"2026-11-18T15:47Z"},{"id":"command-a-03-2025-quality","name":"command-a-03-2025-quality","org":"Cohere","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":12,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"computer-use-preview-2025-03-11","name":"computer-use-preview-2025-03-11","org":"","color":"#64748b","category":"preview","params_b":null,"context_window":8192,"open_weight":false,"cells":[{"note":"computer-use tasks (v1)","label":"OSWorld","trust":2,"value":31.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"os_world","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-03-11","release_source":"name","price_in":3.0,"price_out":12.0,"cheapest_provider":"openai","cheapest_model":"computer-use-preview-2025-03-11","price_observed_at":"2026-11-18T15:47Z"},{"id":"darwin-180b-rsi","name":"Darwin-180B-RSI","org":"FINAL-Bench","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":88.12,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"darwin-27b-opus","name":"Darwin-27B-Opus","org":"FINAL-Bench","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":86.9,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"darwin-36b-opus","name":"Darwin-36B-Opus","org":"FINAL-Bench","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":88.4,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"darwin-397b-ztc","name":"Darwin-397B-ZTC","org":"FINAL-Bench","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":93.43,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"darwin-398b-jgos","name":"Darwin-398B-JGOS","org":"FINAL-Bench","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":90.9,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"davinci-002","name":"davinci-002","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":16384,"open_weight":false,"cells":[{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":0.144,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":2.0,"price_out":2.0,"cheapest_provider":"openai","cheapest_model":"davinci-002","price_observed_at":"2026-11-18T15:47Z"},{"id":"dbrx-instruct","name":"dbrx-instruct","org":"Databricks","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":32.891,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-03-27","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-chat-v2-0628","name":"DeepSeek Chat V2 0628","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":69.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-chat-v3-prev","name":"DeepSeek Chat V3 (prev)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":48.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2024-12-26","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-coder-33b","name":"DeepSeek Coder 33B","org":"DeepSeek,Peking University","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":75.93,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-11-02","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-coder-6-7b","name":"DeepSeek Coder 6.7B","org":"DeepSeek,Peking University","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":67.02,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-11-02","release_source":"epoch","price_in":0.06,"price_out":0.12,"cheapest_provider":"llamagate","cheapest_model":"deepseek-coder-6.7b","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-r1-05-28","name":"Deepseek R1 (05/28)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.12,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.4,"price_out":0.4,"cheapest_provider":"hyperbolic","cheapest_model":"deepseek-ai/DeepSeek-R1","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-r1-plus-claude-3-5-sonnet-20241022","name":"DeepSeek R1 + claude-3-5-sonnet-20241022","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":64,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-v3-2-reasoner","name":"DeepSeek V3.2 Reasoner","org":"deepseek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":60,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-v4-unknown","name":"DeepSeek v4 (unknown)","org":"DeepSeek","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":46.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-04-24","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":154.43,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-31","release_source":"epoch","price_in":0.06,"price_out":0.18,"cheapest_provider":"deepinfra","cheapest_model":"deepseek-ai/DeepSeek-V4-Flash-0731","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-flash-0731-low","name":"DeepSeek V4 Flash 0731 (Low)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":45.97222,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-31","release_source":"epoch","price_in":0.06,"price_out":0.18,"cheapest_provider":"deepinfra","cheapest_model":"deepseek-ai/DeepSeek-V4-Flash-0731","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-flash-0731-none","name":"DeepSeek V4 Flash 0731 (None)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":2.0833333,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-31","release_source":"epoch","price_in":0.06,"price_out":0.18,"cheapest_provider":"deepinfra","cheapest_model":"deepseek-ai/DeepSeek-V4-Flash-0731","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":155.96,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-08-13","release_source":"epoch","price_in":0.111,"price_out":4.44,"cheapest_provider":"openrouter","cheapest_model":"deepseek/deepseek-v4-pro-0813","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-pro-0813-low","name":"DeepSeek V4 Pro 0813 (Low)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":56.25,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-08-13","release_source":"epoch","price_in":0.111,"price_out":4.44,"cheapest_provider":"openrouter","cheapest_model":"deepseek/deepseek-v4-pro-0813","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-1-flash-low","name":"DeepSeek V4.1 Flash (Low)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":60.555557,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-09-09","release_source":"epoch","price_in":0.01,"price_out":1.2,"cheapest_provider":"coralbricks","cheapest_model":"deepseek-v4.1-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-1-flash-1017","name":"DeepSeek V4.1 Flash 1017","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":159.4,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-17","release_source":"epoch","price_in":0.036,"price_out":0.5,"cheapest_provider":"openrouter","cheapest_model":"deepseek/deepseek-v4.1-flash-1017","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-1-flash-1017-medium","name":"DeepSeek V4.1 Flash 1017 (medium)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":46.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-17","release_source":"epoch","price_in":0.036,"price_out":0.5,"cheapest_provider":"openrouter","cheapest_model":"deepseek/deepseek-v4.1-flash-1017","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-coder-v2-lite-instruct","name":"DeepSeek-Coder-V2-Lite-Instruct","org":"deepseek-ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":41.57,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-math-7b-instruct","name":"deepseek-math-7b-instruct","org":"deepseek-ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":35.3,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-v2","name":"DeepSeek-V2","org":"deepseek-ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":79.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2024-05-06","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-v2-moe-236b-may-2024","name":"DeepSeek-V2 (MoE-236B, May 2024)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":120.76,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-05-06","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-v2-chat","name":"DeepSeek-V2-Chat","org":"deepseek-ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":54.81,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2024-05-06","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-v2-5-1210","name":"DeepSeek-V2.5-1210","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":72.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepseek-v3-2-thinking-fireworks","name":"DeepSeek-V3.2 (Thinking; Fireworks)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"writing mathematical proofs","label":"ProofBench","trust":2,"value":8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"proofbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.26,"price_out":0.38,"cheapest_provider":"deepinfra","cheapest_model":"deepseek-ai/DeepSeek-V3.2","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v3-2-exp-high","name":"DeepSeek-V3.2-Exp (high)","org":"DeepSeek","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":83.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.27,"price_out":0.41,"cheapest_provider":"gmi","cheapest_model":"deepseek-ai/DeepSeek-V3.2-Exp","price_observed_at":"2026-11-18T15:47Z"},{"id":"deepseek-v4-pro-high-20260813","name":"deepseek-v4-pro-high-20260813","org":"DeepSeek","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1583,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-07","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-06)","higher_is_better":true}],"n_cells":1,"released_at":"2026-08-13","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"deepswe-preview","name":"DeepSWE-Preview","org":"","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":42.2,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"devstral-2512","name":"Devstral (2512)","org":"mistral","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":53.8,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.4,"price_out":2.0,"cheapest_provider":"mistral","cheapest_model":"devstral-latest","price_observed_at":"2026-11-18T15:47Z"},{"id":"devstral-small-2512","name":"Devstral Small (2512)","org":"mistral","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":56.4,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.07,"price_out":0.28,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"mistral/devstral-small","price_observed_at":"2026-11-18T15:47Z"},{"id":"devstral-small-2505","name":"DevStral Small 2505","org":"Mistral","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":46.8,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-05-21","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"devstral-small-2507","name":"DevStral Small 2507","org":"Mistral","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":38,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-07-10","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"diffbot-small-xl","name":"diffbot-small-xl","org":"","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1023,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"dola-seed-2-0-pro","name":"dola-seed-2.0-pro","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1257,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"dolly-2-0-12b","name":"Dolly 2.0-12b","org":"Databricks","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":55.1,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-04-11","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"dots3-note-preview-max","name":"Dots3-Note Preview (Max)","org":"","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":76.80556,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"dracarys2-72b-instruct","name":"Dracarys2-72B-Instruct","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":66.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ernie-4-5-21b-a3b-base-pt","name":"ERNIE-4.5-21B-A3B-Base-PT","org":"baidu","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":56.7,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ernie-4-5-300b-a47b-base-pt","name":"ERNIE-4.5-300B-A47B-Base-PT","org":"baidu","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":69.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ernie-4-5-300b-a47b-pt","name":"ERNIE-4.5-300B-A47B-PT","org":"baidu","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":78.4,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"eurus-2-7b-prime","name":"Eurus-2-7B-PRIME","org":"Tsinghua University,University of Illinois Urbana-Champaign (UIUC),Shanghai AI L","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":33.902,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-12-31","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"exaone-3-5-2-4b-instruct","name":"EXAONE-3.5-2.4B-Instruct","org":"LGAI-EXAONE","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":39.1,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"exaone-3-5-32b-instruct","name":"EXAONE-3.5-32B-Instruct","org":"LGAI-EXAONE","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":58.91,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"exaone-3-5-7-8b-instruct","name":"EXAONE-3.5-7.8B-Instruct","org":"LGAI-EXAONE","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":46.24,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"falcon-2-11b","name":"Falcon 2 11B","org":"Technology Innovation Institute","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":99.01,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-05-09","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"falcon-180b","name":"Falcon-180B","org":"Technology Innovation Institute","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":106.92,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-09-06","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"falcon-40b","name":"Falcon-40B","org":"Technology Innovation Institute","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":90.42,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-05-25","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"falcon-7b","name":"Falcon-7B","org":"Technology Innovation Institute","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":72.84,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-04-24","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"frogboss-32b-2510","name":"Frogboss 32B 2510","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":53.6,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"frogmini-14b-2510","name":"Frogmini 14B 2510","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":45,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-1-5-flash-sep-2024","name":"Gemini 1.5 Flash (Sep 2024)","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":126.96,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-09-24","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-1-5-pro","name":"Gemini 1.5 Pro","org":"Google","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0.8,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2024-02-15","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-1-5-pro-may-2024","name":"Gemini 1.5 Pro (May 2024)","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":125.37,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-05-14","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-1-5-pro-sept-2024","name":"Gemini 1.5 Pro (Sept 2024)","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":130.45,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-09-24","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-0-flash-experimental","name":"Gemini 2.0 Flash (Experimental)","org":"Google DeepMind","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":44.2,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.15,"price_out":0.6,"cheapest_provider":"vertex_ai","cheapest_model":"gemini-2.0-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-0-flash-thinking-jan-2025","name":"Gemini 2.0 Flash Thinking (Jan 2025)","org":"Google DeepMind,Google","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":133.61,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-01-21","release_source":"epoch","price_in":0.15,"price_out":0.6,"cheapest_provider":"vertex_ai","cheapest_model":"gemini-2.0-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-0-pro","name":"Gemini 2.0 Pro","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":135.41,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-02-05","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-0-pro-exp-02-05","name":"Gemini 2.0 Pro exp-02-05","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":35.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-02-05","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-5-flash-apr-2025","name":"Gemini 2.5 Flash (Apr 2025)","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":140.73,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-04-17","release_source":"epoch","price_in":0.3,"price_out":2.5,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-flash-jun-2025","name":"Gemini 2.5 Flash (Jun 2025)","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":140.84,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-06-17","release_source":"epoch","price_in":0.3,"price_out":2.5,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-flash-may-2025","name":"Gemini 2.5 Flash (May 2025)","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":141.58,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-05-20","release_source":"epoch","price_in":0.3,"price_out":2.5,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-flash-preview","name":"Gemini 2.5 Flash (Preview)","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.69,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.3,"price_out":2.5,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-flash-preview-thinking-16k","name":"Gemini 2.5 Flash (Preview) (Thinking 16K)","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.98,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.3,"price_out":2.5,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-flash-preview-thinking-1k","name":"Gemini 2.5 Flash (Preview) (Thinking 1K)","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":2.16,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.3,"price_out":2.5,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-flash-preview-thinking-24k","name":"Gemini 2.5 Flash (Preview) (Thinking 24K)","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":2.54,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.3,"price_out":2.5,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-flash-preview-thinking-8k","name":"Gemini 2.5 Flash (Preview) (Thinking 8K)","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":2.12,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.3,"price_out":2.5,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-flash-sep-2025","name":"Gemini 2.5 Flash (Sep 2025)","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":141.94,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-09-25","release_source":"epoch","price_in":0.3,"price_out":2.5,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-flash-lite-jun-2025","name":"Gemini 2.5 Flash-Lite (Jun 2025)","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":133.05,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-06-17","release_source":"epoch","price_in":0.1,"price_out":0.4,"cheapest_provider":"gemini","cheapest_model":"gemini-2.5-flash-lite","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-pro-jun-2025","name":"Gemini 2.5 Pro (Jun 2025)","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":144.91,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-06-05","release_source":"epoch","price_in":1.25,"price_out":10.0,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-pro-mar-2025","name":"Gemini 2.5 Pro (Mar 2025)","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":144.05,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-03-31","release_source":"epoch","price_in":1.25,"price_out":10.0,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-pro-may-2025","name":"Gemini 2.5 Pro (May 2025)","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":140.64,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-05-06","release_source":"epoch","price_in":1.25,"price_out":10.0,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-pro-thinking-16k","name":"Gemini 2.5 Pro (Thinking 16K)","org":"Google","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":4.03,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":1.25,"price_out":10.0,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-pro-thinking-1k","name":"Gemini 2.5 Pro (Thinking 1K)","org":"Google","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":1.25,"price_out":10.0,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-pro-thinking-32k","name":"Gemini 2.5 Pro (Thinking 32K)","org":"Google","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":4.86,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":1.25,"price_out":10.0,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-pro-thinking-8k","name":"Gemini 2.5 Pro (Thinking 8K)","org":"Google","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":2.92,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":1.25,"price_out":10.0,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-pro-preview-03-25","name":"Gemini 2.5 Pro Preview 03-25","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":72.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-03-25","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-5-pro-preview-05-06","name":"Gemini 2.5 Pro Preview 05-06","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":76.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-05-06","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-3-deep-think","name":"Gemini 3 Deep Think","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":25.714,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-02-12","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-3-deep-think-2-26","name":"Gemini 3 Deep Think (2/26)","org":"Google","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":84.58,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-3-deep-think-preview","name":"Gemini 3 Deep Think (Preview) \u00b2","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":45.14,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-3-flash-preview-high","name":"Gemini 3 Flash Preview (High)","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":33.61,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.5,"price_out":3.0,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3-flash-preview","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-flash-preview-low","name":"Gemini 3 Flash Preview (Low)","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.25,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.5,"price_out":3.0,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3-flash-preview","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-flash-preview-medium","name":"Gemini 3 Flash Preview (Medium)","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":12.78,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.5,"price_out":3.0,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3-flash-preview","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-flash-preview-minimal","name":"Gemini 3 Flash Preview (Minimal)","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":3.33,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.5,"price_out":3.0,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3-flash-preview","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-1-pro","name":"Gemini 3.1 Pro","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":156.25,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-02-19","release_source":"epoch","price_in":2.0,"price_out":12.0,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3.1-pro-preview","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-5-flash-minimal","name":"Gemini 3.5 Flash (Minimal)","org":"Google","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":8.89,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-05-19","release_source":"epoch","price_in":1.5,"price_out":9.0,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-3.5-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-5-flash-lite-high","name":"Gemini 3.5 Flash-Lite (High)","org":"Google","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":10.277778,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-21","release_source":"epoch","price_in":0.3,"price_out":2.5,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3.5-flash-lite","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-5-flash-lite-low","name":"Gemini 3.5 Flash-Lite (Low)","org":"Google","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.5277778,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-21","release_source":"epoch","price_in":0.3,"price_out":2.5,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3.5-flash-lite","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-5-flash-lite-medium","name":"Gemini 3.5 Flash-Lite (Medium)","org":"Google","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":5.2777777,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-21","release_source":"epoch","price_in":0.3,"price_out":2.5,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3.5-flash-lite","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-5-flash-lite-minimal","name":"Gemini 3.5 Flash-Lite (Minimal)","org":"Google","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0.8333333,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-21","release_source":"epoch","price_in":0.3,"price_out":2.5,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3.5-flash-lite","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-6-flash-low","name":"Gemini 3.6 Flash (Low)","org":"Google","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":30.416666,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-21","release_source":"epoch","price_in":0.75,"price_out":3.75,"cheapest_provider":"gemini","cheapest_model":"gemini-3.6-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-6-flash-medium","name":"Gemini 3.6 Flash (Medium)","org":"Google","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":50.416668,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-21","release_source":"epoch","price_in":0.75,"price_out":3.75,"cheapest_provider":"gemini","cheapest_model":"gemini-3.6-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-6-flash-minimal","name":"Gemini 3.6 Flash (Minimal)","org":"Google","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":2.6388888,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-21","release_source":"epoch","price_in":0.75,"price_out":3.75,"cheapest_provider":"gemini","cheapest_model":"gemini-3.6-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-7-flash","name":"Gemini 3.7 Flash","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":1048576,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":158.38,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-08-13","release_source":"epoch","price_in":0.75,"price_out":3.75,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3.7-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-4-argon-minimal","name":"Gemini 4 Argon (minimal)","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":48.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-09-30","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"gemini","cheapest_model":"gemini-4-argon","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-4-carbon","name":"Gemini 4 Carbon","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":169.78,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-28","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"gemini","cheapest_model":"gemini-4-carbon","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-4-hydrogen-low","name":"Gemini 4 Hydrogen (low)","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":27.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-11-12","release_source":"epoch","price_in":1.5,"price_out":7.5,"cheapest_provider":"gemini","cheapest_model":"gemini-4-hydrogen","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-4-hydrogen-medium","name":"Gemini 4 Hydrogen (medium)","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":46.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-11-12","release_source":"epoch","price_in":1.5,"price_out":7.5,"cheapest_provider":"gemini","cheapest_model":"gemini-4-hydrogen","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-robotics-er-1-6-preview","name":"Gemini Robotics-ER 1.6 Preview","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"reading and reasoning over floor plans","label":"Blueprint-Bench 2","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"blueprint_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-1-5-flash-002-0924","name":"gemini-1.5-flash-002 (0924)","org":"Google","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":51.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-1-5-flash-8b-exp-0827","name":"gemini-1.5-flash-8b-exp-0827","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":38.3,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-1-5-flash-8b-exp-0924","name":"gemini-1.5-flash-8b-exp-0924","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":38.3,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-1-5-flash-exp-0827","name":"gemini-1.5-flash-exp-0827","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":52.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-1-5-flash-latest","name":"gemini-1.5-flash-latest","org":"Google","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":44.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2024-05-14","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-1-5-pro-001-feb24","name":"gemini-1.5-pro-001-feb24","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"cybersecurity capture-the-flag tasks","label":"Cybench","trust":2,"value":7.5,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cybench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-02-15","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-1-5-pro-exp-0827","name":"gemini-1.5-pro-exp-0827","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":66.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-0-flash-02-05","name":"gemini-2.0-flash-02-05","org":"Google DeepMind,Google","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":0.63,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-02-05","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-0-flash-lite","name":"gemini-2.0-flash-lite","org":"Google DeepMind","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":52.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-02-05","release_source":"epoch","price_in":0.075,"price_out":0.3,"cheapest_provider":"vertex_ai","cheapest_model":"gemini-2.0-flash-lite","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-0-flash-thinking-exp-01-21","name":"gemini-2.0-flash-thinking-exp-01-21","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":18.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-01-21","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-5-flash-lite-preview-06-17-16k","name":"gemini-2.5-flash-lite-preview-06-17_16K","org":"Google DeepMind","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":35.22,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-06-17","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-5-flash-preview-04-17","name":"gemini-2.5-flash-preview-04-17","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":47.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-04-17","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-5-flash-preview-04-17-16k-thinking","name":"gemini-2.5-flash-preview-04-17 (16K thinking)","org":"Google DeepMind","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":40.95,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-04-17","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-5-flash-preview-05-20-16k","name":"gemini-2.5-flash-preview-05-20_16K","org":"Google DeepMind","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":40.95,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-05-20","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-5-flash-preview-09-2025-16k","name":"gemini-2.5-flash-preview-09-2025_16k","org":"Google DeepMind","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":41.91,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-5-pro-grounding","name":"gemini-2.5-pro-grounding","org":"Google","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1142,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-5-pro-preview-06-05","name":"gemini-2.5-pro-preview-06-05","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":83.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-06-05","release_source":"name","price_in":1.25,"price_out":10.0,"cheapest_provider":"openrouter","cheapest_model":"google/gemini-2.5-pro-preview","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-2-5-pro-16k","name":"gemini-2.5-pro_16K","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":54.03,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-06-17","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-2-5-pro-32k","name":"gemini-2.5-pro_32K","org":"Google DeepMind","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":785.52,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-03-25","release_source":"curated","price_in":1.25,"price_out":10.0,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-2.5-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-flash-thinking-minimal","name":"gemini-3-flash (thinking-minimal)","org":"Google","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1260,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.5,"price_out":3.0,"cheapest_provider":"aihubmix","cheapest_model":"gemini-3-flash-preview","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-flash-grounding","name":"gemini-3-flash-grounding","org":"Google","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1198,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-3-pro-grounding","name":"gemini-3-pro-grounding","org":"Google","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1207,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-3-1-flash-lite-high","name":"gemini-3.1-flash-lite (high)","org":"Google","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"tool use across 1,000 MCP tasks","label":"MCP Atlas","trust":4,"value":57.1,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/mcp_atlas","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"mcp_atlas","source_label":"Scale Labs MCP Atlas leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-03-03","release_source":"epoch","price_in":0.25,"price_out":1.5,"cheapest_provider":"deepinfra","cheapest_model":"google/gemini-3.1-flash-lite","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-3-1-pro-grounding","name":"gemini-3.1-pro-grounding","org":"Google","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1210,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-4-argon","name":"gemini-4-argon","org":"Google","color":"#64748b","category":"frontier","params_b":null,"context_window":1048576,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":166.57,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-09-30","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"gemini","cheapest_model":"gemini-4-argon","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemini-exp-1114","name":"gemini-exp-1114","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":60.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-exp-1121","name":"gemini-exp-1121","org":"Google","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":57.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemini-gemini-1-5-pro-latest","name":"gemini/gemini-1.5-pro-latest","org":"Google","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Refactoring","trust":4,"value":49.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/refactor.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_refactor","source_label":"Aider Aider Refactoring leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2024-02-15","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemma-2b","name":"Gemma 2B","org":"Google DeepMind","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":75.36,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-02-21","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemma-3-27b","name":"Gemma 3 27B","org":"Google DeepMind","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":127.5,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-03-12","release_source":"epoch","price_in":0.08,"price_out":0.45,"cheapest_provider":"openrouter","cheapest_model":"google/gemma-3-27b-it","price_observed_at":"2026-11-18T15:47Z"},{"id":"gemma-7b","name":"Gemma 7B","org":"Google DeepMind","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":102.87,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-02-21","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemma-3-270m-it","name":"gemma-3-270m-it","org":"Google DeepMind","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-08-14","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gemma2-27b-instruct-q8-0","name":"gemma2:27b-instruct-q8_0","org":"Google","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":36.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"github-copilot-plus-copilot-fusion","name":"GitHub Copilot + Copilot Fusion","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":71.21,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"glm-4-9b","name":"glm-4-9b","org":"zai-org","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":47.92,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"glm-4-5-thinking","name":"GLM-4.5 Thinking","org":"Z.ai (Zhipu AI),Tsinghua University","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":40.58,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.4,"price_out":1.6,"cheapest_provider":"deepinfra","cheapest_model":"zai-org/GLM-4.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-4-5-fp8","name":"GLM-4.5-FP8","org":"Z.ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":58.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-07-27","release_source":"epoch","price_in":0.6,"price_out":2.2,"cheapest_provider":"gmi","cheapest_model":"zai-org/GLM-4.5-FP8","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-4-6-fp8","name":"GLM-4.6-FP8","org":"zai-org","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":81.31313,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"glm-4-7-fp8","name":"GLM-4.7-FP8","org":"zai-org","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":87.878784,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.4,"price_out":2.0,"cheapest_provider":"gmi","cheapest_model":"zai-org/GLM-4.7-FP8","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-4p7","name":"glm-4p7","org":"Z.ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"tool use across 1,000 MCP tasks","label":"MCP Atlas","trust":4,"value":58.1,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/mcp_atlas","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"mcp_atlas","source_label":"Scale Labs MCP Atlas leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.6,"price_out":2.2,"cheapest_provider":"fireworks_ai","cheapest_model":"glm-4p7","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-5-fp8","name":"GLM-5-FP8","org":"zai-org","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":85.35354,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.6,"price_out":1.92,"cheapest_provider":"gmi","cheapest_model":"zai-org/GLM-5-FP8","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-5-1-none","name":"glm-5.1 (none)","org":"Z.ai (Zhipu AI)","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"research-level maths, tiers 1-3","label":"FrontierMath T1-3","trust":2,"value":24.912,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiermath_t13","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-04-07","release_source":"epoch","price_in":0.825,"price_out":3.301,"cheapest_provider":"empiriolabs","cheapest_model":"glm-5-1","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-5-1-fp8","name":"GLM-5.1-FP8","org":"zai-org","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":83.333336,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":1.4,"price_out":4.4,"cheapest_provider":"gmi","cheapest_model":"zai-org/GLM-5.1-FP8","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-5-1-mxfp4-mixed-ct-autoround","name":"GLM-5.1-MXFP4-Mixed-CT-AutoRound","org":"INCModel3","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":86.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"glm-5-2-high","name":"GLM-5.2 (high)","org":"Z.ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"fresh, decontaminated GitHub-issue tasks (resolved %)","label":"SWE-rebench","trust":2,"value":62.9,"score_max":100,"source_url":"https://swe-rebench.com/","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_rebench","source_label":"SWE-rebench leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-06-16","release_source":"epoch","price_in":0.007,"price_out":6.8,"cheapest_provider":"openrouter","cheapest_model":"z-ai/glm-5.2","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-5-2-minimal","name":"glm-5.2 (minimal)","org":"Z.ai (Zhipu AI)","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":18,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-06-16","release_source":"epoch","price_in":0.007,"price_out":6.8,"cheapest_provider":"openrouter","cheapest_model":"z-ai/glm-5.2","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-5-3","name":"GLM-5.3","org":"Z.ai (Zhipu AI)","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":157.1,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-08-14","release_source":"epoch","price_in":0.0699,"price_out":1.4,"cheapest_provider":"openrouter","cheapest_model":"z-ai/glm-5.3","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-5p1","name":"glm-5p1","org":"Z.ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"tool use across 1,000 MCP tasks","label":"MCP Atlas","trust":4,"value":75.6,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/mcp_atlas","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"mcp_atlas","source_label":"Scale Labs MCP Atlas leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":1.4,"price_out":4.4,"cheapest_provider":"fireworks_ai","cheapest_model":"glm-5p1","price_observed_at":"2026-11-18T15:47Z"},{"id":"glm-5v-turbo","name":"glm-5v-turbo","org":"Z.ai","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted questions about documents","label":"LMArena Document Elo","trust":2,"value":1416,"score_max":2000,"source_url":"https://arena.ai/leaderboard/document","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_document","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":"2026-04-01","release_source":"openrouter","price_in":0.7042,"price_out":3.09848,"cheapest_provider":"aihubmix","cheapest_model":"glm-5v-turbo","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-3-5-turbo-jun-2023","name":"GPT-3.5 Turbo (Jun 2023)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":105.84,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-06-13","release_source":"epoch","price_in":0.5,"price_out":1.5,"cheapest_provider":"azure","cheapest_model":"gpt-3.5-turbo","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-3-5-turbo-0125","name":"gpt-3.5-turbo-0125","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":16385,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":50.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.5,"price_out":1.5,"cheapest_provider":"openai","cheapest_model":"gpt-3.5-turbo-0125","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-3-5-turbo-0301","name":"gpt-3.5-turbo-0301","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":57.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-3-5-turbo-0613","name":"gpt-3.5-turbo-0613","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":50.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-3-5-turbo-1106","name":"gpt-3.5-turbo-1106","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":16385,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":56.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":1.0,"price_out":2.0,"cheapest_provider":"openai","cheapest_model":"gpt-3.5-turbo-1106","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-3-5-turbo-instruct","name":"gpt-3.5-turbo-instruct","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":8192,"open_weight":false,"cells":[{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":0.599,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-09-18","release_source":"epoch","price_in":1.5,"price_out":2.0,"cheapest_provider":"openai","cheapest_model":"gpt-3.5-turbo-instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4-0125","name":"GPT-4 (0125)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":2,"value":19,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard (unverified)","higher_is_better":true}],"n_cells":1,"released_at":"2024-01-25","release_source":"name","price_in":30.0,"price_out":60.0,"cheapest_provider":"openai","cheapest_model":"gpt-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4-0806","name":"GPT-4 (0806)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":2,"value":39.67,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard (unverified)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":30.0,"price_out":60.0,"cheapest_provider":"openai","cheapest_model":"gpt-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4-turbo-apr-2024","name":"GPT-4 Turbo (Apr 2024)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":125.82,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-04-09","release_source":"epoch","price_in":10.0,"price_out":30.0,"cheapest_provider":"openai","cheapest_model":"gpt-4-turbo","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4-turbo-nov-2023","name":"GPT-4 Turbo (Nov 2023)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":124.75,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-01-25","release_source":"epoch","price_in":10.0,"price_out":30.0,"cheapest_provider":"openai","cheapest_model":"gpt-4-turbo","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-4-0314","name":"gpt-4-0314","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":66.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-4-5-preview","name":"gpt-4.5-preview","org":"OpenAI","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":44.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-4o-2024-05-13","name":"gpt-4o-2024-05-13","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":128000,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":72.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2024-05-13","release_source":"name","price_in":5.0,"price_out":15.0,"cheapest_provider":"azure","cheapest_model":"gpt-4o-2024-05-13","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-nano-unknown-thinking","name":"GPT-5 nano (unknown thinking)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":21.798,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.05,"price_out":0.4,"cheapest_provider":"azure","cheapest_model":"gpt-5-nano","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-chat","name":"gpt-5-chat","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":128000,"open_weight":false,"cells":[{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":39.77,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-08-07","release_source":"curated","price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-5-chat","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-codex-high","name":"GPT-5-codex (high)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":54.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-09-15","release_source":"epoch","price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-5-codex","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-search","name":"gpt-5-search","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1133,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-5-1-thinking-high","name":"GPT-5.1 (Thinking, High)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":17.64,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"global/gpt-5.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-1-thinking-low","name":"GPT-5.1 (Thinking, Low)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.94,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"global/gpt-5.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-1-thinking-medium","name":"GPT-5.1 (Thinking, Medium)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":6.53,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"global/gpt-5.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-1-thinking-none","name":"GPT-5.1 (Thinking, None)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0.42,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"global/gpt-5.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-1-codex-max-xhigh","name":"GPT-5.1-Codex-Max (xhigh)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":1228.25,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-11-19","release_source":"epoch","price_in":1.25,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-5.1-codex-max","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-1-search","name":"gpt-5.1-search","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1199,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-5-2-search","name":"gpt-5.2-search","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1207,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-5-2-search-non-reasoning","name":"gpt-5.2-search-non-reasoning","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1172,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-5-3-codex-high","name":"GPT-5.3 Codex (high)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":74.793,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-02-05","release_source":"epoch","price_in":1.75,"price_out":14.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.3-codex","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4-mini","name":"GPT-5.4 Mini","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":272000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":149.35,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-03-17","release_source":"epoch","price_in":0.75,"price_out":4.5,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4-mini-unknown-thinking","name":"GPT-5.4 mini (unknown thinking)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":27.04,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.75,"price_out":4.5,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4-nano","name":"GPT-5.4 Nano","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":272000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":146.33,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-03-17","release_source":"epoch","price_in":0.2,"price_out":1.25,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.4-nano","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-4-search","name":"gpt-5.4-search","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1197,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-5-5-search","name":"gpt-5.5-search","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1242,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-5-5-xhigh-codex-harness","name":"gpt-5.5-xhigh (codex-harness)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1512,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true}],"n_cells":1,"released_at":"2026-04-23","release_source":"epoch","price_in":5.0,"price_out":30.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-luna","name":"GPT-5.6 Luna","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":922000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":156.7,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-09","release_source":"epoch","price_in":0.2,"price_out":1.2,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.6-luna","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-luna-2026-07-30-high","name":"GPT-5.6 Luna 2026-07-30 (High)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":33.333332,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-30","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-5-6-luna-2026-07-30-low","name":"GPT-5.6 Luna 2026-07-30 (Low)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":3.6111112,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-30","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-5-6-luna-2026-07-30-max","name":"GPT-5.6 Luna 2026-07-30 (Max)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":59.583332,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-30","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-5-6-luna-2026-07-30-medium","name":"GPT-5.6 Luna 2026-07-30 (Medium)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":7.361111,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-30","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-5-6-luna-2026-07-30-none","name":"GPT-5.6 Luna 2026-07-30 (None)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0.8333333,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-30","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-5-6-luna-2026-07-30-xhigh","name":"GPT-5.6 Luna 2026-07-30 (XHigh)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":43.47222,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-30","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt-5-6-sol","name":"GPT-5.6 Sol","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":922000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":163.05,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-09","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"replicate","cheapest_model":"openai/gpt-5.6-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-terra","name":"GPT-5.6 Terra","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":922000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":160.78,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-09","release_source":"epoch","price_in":2.0,"price_out":12.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.6-terra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-luna-xhigh-codex-harness","name":"gpt-5.6-luna-xhigh (codex-harness)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1518,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-09","release_source":"epoch","price_in":0.2,"price_out":1.2,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.6-luna","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-sol-xhigh-codex-harness","name":"gpt-5.6-sol-xhigh (codex-harness)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1620,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-07","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-06)","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-09","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"replicate","cheapest_model":"openai/gpt-5.6-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-5-6-terra-xhigh-codex-harness","name":"gpt-5.6-terra-xhigh (codex-harness)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1519,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-09","release_source":"epoch","price_in":2.0,"price_out":12.0,"cheapest_provider":"aihubmix","cheapest_model":"gpt-5.6-terra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-luna","name":"gpt-6-luna","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":922000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":156.39,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-09-22","release_source":"epoch","price_in":0.1,"price_out":0.5,"cheapest_provider":"azure","cheapest_model":"gpt-6-luna","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-1-sol","name":"gpt-6.1-sol","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":922000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":168.63,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-09-29","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.1-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-1-terra","name":"gpt-6.1-terra","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":922000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":167.72,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-26","release_source":"epoch","price_in":1.0,"price_out":5.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.1-terra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-2-astra-high","name":"GPT-6.2 Astra (High)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":94.583336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-11-17","release_source":"epoch","price_in":8.0,"price_out":40.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.2-astra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-2-astra-low","name":"GPT-6.2 Astra (Low)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":91.666664,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-11-17","release_source":"epoch","price_in":8.0,"price_out":40.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.2-astra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-2-astra-medium","name":"GPT-6.2 Astra (Medium)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":93.333336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-11-17","release_source":"epoch","price_in":8.0,"price_out":40.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.2-astra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-2-astra-xhigh","name":"GPT-6.2 Astra (XHigh)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":95.833336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-11-17","release_source":"epoch","price_in":8.0,"price_out":40.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.2-astra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-2-sol-high","name":"GPT-6.2 Sol (High)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":85.833336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-11-17","release_source":"openrouter","price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.2-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-2-sol-low","name":"GPT-6.2 Sol (Low)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":57.916668,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-11-17","release_source":"openrouter","price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.2-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-2-sol-max","name":"GPT-6.2 Sol (Max)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":85.833336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-11-17","release_source":"openrouter","price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.2-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-2-sol-medium","name":"GPT-6.2 Sol (Medium)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":82.916664,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-11-17","release_source":"openrouter","price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.2-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-2-sol-xhigh","name":"GPT-6.2 Sol (XHigh)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":85.833336,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-11-17","release_source":"openrouter","price_in":2.0,"price_out":10.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.2-sol","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-6-2-astra","name":"gpt-6.2-astra","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":922000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":174.17,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-11-17","release_source":"epoch","price_in":8.0,"price_out":40.0,"cheapest_provider":"azure","cheapest_model":"gpt-6.2-astra","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt-oss-120b-unknown-thinking","name":"gpt-oss-120b (unknown thinking)","org":"OpenAI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"real terminal tasks (agent + model)","label":"Terminal-Bench 2.0","trust":2,"value":18.689,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"terminal_bench_2","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.03,"price_out":0.17,"cheapest_provider":"wandb","cheapest_model":"openai/gpt-oss-120b","price_observed_at":"2026-11-18T15:47Z"},{"id":"gpt2-xl","name":"gpt2-xl","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":0.054,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2019-11-05","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"gpt4-base-10-shot","name":"GPT4 base 10-shot","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"commonsense sentence completion","label":"HellaSwag","trust":4,"value":95.3,"score_max":100,"source_url":"https://rowanzellers.com/hellaswag/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"hellaswag","source_label":"HellaSwag official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"granite-3-0-2b-base","name":"granite-3.0-2b-base","org":"ibm-granite","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":21.72,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"granite-3-0-8b-base","name":"granite-3.0-8b-base","org":"ibm-granite","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":31.03,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"granite-3-1-1b-a400m-base","name":"granite-3.1-1b-a400m-base","org":"ibm-granite","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":12.34,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"granite-3-1-2b-base","name":"granite-3.1-2b-base","org":"ibm-granite","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":23.89,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"granite-3-1-3b-a800m-base","name":"granite-3.1-3b-a800m-base","org":"ibm-granite","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":20.39,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"granite-3-1-8b-base","name":"granite-3.1-8b-base","org":"ibm-granite","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":33.08,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"grok-3-mini-low","name":"Grok 3 Mini (Low)","org":"xAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0.42,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-02-19","release_source":"curated","price_in":0.25,"price_out":1.27,"cheapest_provider":"azure_ai","cheapest_model":"grok-3-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-3-mini-beta","name":"Grok 3 Mini Beta","org":"xAI","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":49.3,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"grok-4-fast-reasoning","name":"Grok 4 (Fast Reasoning)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":5.28,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":3.0,"price_out":15.0,"cheapest_provider":"azure_ai","cheapest_model":"grok-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-thinking","name":"Grok 4 (Thinking)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":15.975,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":3.0,"price_out":15.0,"cheapest_provider":"azure_ai","cheapest_model":"grok-4","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-20-reasoning","name":"Grok 4.20 (Reasoning)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":65.14,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-02-17","release_source":"epoch","price_in":1.25,"price_out":2.5,"cheapest_provider":"openrouter","cheapest_model":"x-ai/grok-4.20","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-3-no-thinking","name":"Grok 4.3 (no thinking)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":1.25,"price_out":2.5,"cheapest_provider":"azure_ai","cheapest_model":"grok-4.3","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-3-beta","name":"Grok 4.3 Beta","org":"xAI","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":149.91,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"grok-4-5-low","name":"Grok 4.5 (Low)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":33.06,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-08","release_source":"epoch","price_in":2.0,"price_out":6.0,"cheapest_provider":"github_copilot","cheapest_model":"grok-4.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-5-medium","name":"Grok 4.5 (Medium)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":52.64,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-08","release_source":"epoch","price_in":2.0,"price_out":6.0,"cheapest_provider":"github_copilot","cheapest_model":"grok-4.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-6","name":"Grok 4.6","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":157.82,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-08-12","release_source":"curated","price_in":2.0,"price_out":6.0,"cheapest_provider":"aihubmix","cheapest_model":"grok-4.6","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-build-plus-grok-4-5","name":"Grok Build + Grok 4.5","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":12.42,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"grok-build-plus-grok-4-6","name":"Grok Build + Grok 4.6","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":19.7,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"grok-build-plus-grok-4-7","name":"Grok Build + Grok 4.7","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":38.79,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"grok-2","name":"Grok-2","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":58.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2024-12-12","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"xai/grok-2","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-2-dec-2024","name":"Grok-2 (Dec 2024)","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":129.75,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-12-12","release_source":"epoch","price_in":2.0,"price_out":10.0,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"xai/grok-2","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-2-mini","name":"Grok-2-mini","org":"xAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":54.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"grok-3-mini-beta-medium","name":"grok-3-mini-beta (medium)","org":"xAI","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":66.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"grok-4-1-fast-non-reasoning","name":"grok-4-1-fast-non-reasoning","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"trick questions that test common sense","label":"SimpleBench","trust":2,"value":56,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"simplebench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.2,"price_out":0.5,"cheapest_provider":"azure_ai","cheapest_model":"grok-4-1-fast-non-reasoning","price_observed_at":"2026-11-18T15:47Z"},{"id":"grok-4-1-fast-search","name":"grok-4-1-fast-search","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1171,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"grok-4-fast-search","name":"grok-4-fast-search","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1171,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"grok-4-search","name":"grok-4-search","org":"xAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1142,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"hunyuan-a13b-instruct","name":"Hunyuan-A13B-Instruct","org":"tencent","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":67.3,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-07-08","release_source":"openrouter","price_in":0.14,"price_out":0.57,"cheapest_provider":"openrouter","cheapest_model":"tencent/hunyuan-a13b-instruct","price_observed_at":"2026-11-15T12:16Z"},{"id":"hy3","name":"Hy3","org":"tencent","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":90.4,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-06","release_source":"openrouter","price_in":0.132,"price_out":0.528,"cheapest_provider":"openrouter","cheapest_model":"tencent/hy3","price_observed_at":"2026-11-18T15:47Z"},{"id":"hy3-preview","name":"Hy3-preview","org":"tencent","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":87.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.132,"price_out":0.528,"cheapest_provider":"openrouter","cheapest_model":"tencent/hy3","price_observed_at":"2026-11-18T15:47Z"},{"id":"illada-8b-instruct","name":"iLLaDA-8B-Instruct","org":"GSAI-ML","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":52.3,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"inkling-high","name":"Inkling (high)","org":"Thinking Machines","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":32.29,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-15","release_source":"epoch","price_in":0.95,"price_out":4.05,"cheapest_provider":"deepinfra","cheapest_model":"thinkingmachines/Inkling","price_observed_at":"2026-11-18T15:47Z"},{"id":"inkling-small-high","name":"Inkling Small (High)","org":"","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":33.055557,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-15","release_source":"epoch","price_in":0.45,"price_out":1.2,"cheapest_provider":"deepinfra","cheapest_model":"thinkingmachines/Inkling-Small","price_observed_at":"2026-11-18T15:47Z"},{"id":"inkling-small-low","name":"Inkling Small (Low)","org":"","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":4.861111,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-15","release_source":"epoch","price_in":0.45,"price_out":1.2,"cheapest_provider":"deepinfra","cheapest_model":"thinkingmachines/Inkling-Small","price_observed_at":"2026-11-18T15:47Z"},{"id":"inkling-small-medium","name":"Inkling Small (Medium)","org":"","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":13.611111,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-15","release_source":"epoch","price_in":0.45,"price_out":1.2,"cheapest_provider":"deepinfra","cheapest_model":"thinkingmachines/Inkling-Small","price_observed_at":"2026-11-18T15:47Z"},{"id":"inkling-small-minimal","name":"Inkling Small (Minimal)","org":"","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":4.0277777,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-15","release_source":"epoch","price_in":0.45,"price_out":1.2,"cheapest_provider":"deepinfra","cheapest_model":"thinkingmachines/Inkling-Small","price_observed_at":"2026-11-18T15:47Z"},{"id":"inkling-small-none","name":"Inkling Small (None)","org":"","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-15","release_source":"epoch","price_in":0.45,"price_out":1.2,"cheapest_provider":"deepinfra","cheapest_model":"thinkingmachines/Inkling-Small","price_observed_at":"2026-11-18T15:47Z"},{"id":"inkling-2-small","name":"Inkling-2-Small","org":"thinkingmachines","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":93,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-11-05","release_source":"epoch","price_in":0.3,"price_out":1.0,"cheapest_provider":"nebius","cheapest_model":"thinkingmachines/Inkling-2-Small","price_observed_at":"2026-11-18T15:47Z"},{"id":"intellect-1","name":"INTELLECT-1","org":"Prime Intellect,Hugging Face,Arcee AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":83.09,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-11-29","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"intern-s1","name":"Intern-S1","org":"internlm","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":83.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"intern-s1-pro","name":"Intern-S1-Pro","org":"internlm","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":86.6,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"intern-s2-preview","name":"Intern-S2-Preview","org":"internlm","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":88,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"intern-s2-preview-397b","name":"Intern-S2-Preview-397B","org":"internlm","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":89.75,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"internlm-20b","name":"internlm-20b","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":98.79,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-09-18","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"internlm-7b","name":"internlm-7b","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":84.31,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-07-05","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"internlm2-chat-20b","name":"internlm2-chat-20b","org":"internlm","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":79.6,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"internlm2-math-plus-20b","name":"internlm2-math-plus-20b","org":"internlm","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":37.1,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"internlm2-math-plus-7b","name":"internlm2-math-plus-7b","org":"internlm","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":33.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"internlm2-5-7b-chat","name":"internlm2_5-7b-chat","org":"internlm","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":86,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"internlm3-8b-instruct","name":"internlm3-8b-instruct","org":"internlm","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":57.6,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"kimi-k2-jul-2025","name":"Kimi K2 (Jul 2025)","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":141.25,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-07-12","release_source":"epoch","price_in":0.55,"price_out":2.2,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"moonshotai/kimi-k2","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k2-0905-novita","name":"Kimi K2 0905 (Novita)","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":267.12,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-09-05","release_source":"epoch","price_in":0.6,"price_out":2.5,"cheapest_provider":"novita","cheapest_model":"moonshotai/kimi-k2-0905","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k2-thinking-together","name":"Kimi K2 Thinking (Together)","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"learning new rules from context","label":"CL-bench","trust":2,"value":17.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cl_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.55,"price_out":2.2,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"moonshotai/kimi-k2","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k2-5-instant","name":"Kimi K2.5 (instant)","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"building web apps, crowd-voted (Elo)","label":"WebDev Arena","trust":2,"value":1405.66,"score_max":2000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"webdev_arena","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-01-27","release_source":"epoch","price_in":0.45,"price_out":2.25,"cheapest_provider":"deepinfra","cheapest_model":"moonshotai/Kimi-K2.5","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k3-1-pro-xhigh","name":"Kimi K3.1 Pro (xhigh)","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unusual machine-learning tasks (v3)","label":"WeirdML v3","trust":2,"value":9.89,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml_v3","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-23","release_source":"epoch","price_in":0.2403,"price_out":8.4023,"cheapest_provider":"openrouter","cheapest_model":"moonshotai/kimi-k3.1-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k2-0711-preview","name":"kimi-k2-0711-preview","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unusual machine-learning tasks","label":"WeirdML","trust":2,"value":39.36,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"weirdml","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.57,"price_out":2.3,"cheapest_provider":"openrouter","cheapest_model":"moonshotai/kimi-k2","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k2-6-none","name":"kimi-k2.6 (none)","org":"Moonshot","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":12,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-04-20","release_source":"epoch","price_in":0.38,"price_out":1.6,"cheapest_provider":"clf_ai_gateway","cheapest_model":"kimi-k2.6","price_observed_at":"2026-11-18T15:47Z"},{"id":"kimi-k2p5","name":"kimi-k2p5","org":"Moonshot AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"tool use across 1,000 MCP tasks","label":"MCP Atlas","trust":4,"value":64.4,"score_max":100,"source_url":"https://labs.scale.com/leaderboard/mcp_atlas","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"mcp_atlas","source_label":"Scale Labs MCP Atlas leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.6,"price_out":3.0,"cheapest_provider":"fireworks_ai","cheapest_model":"kimi-k2p5","price_observed_at":"2026-11-18T15:47Z"},{"id":"llada-8b-instruct","name":"LLaDA-8B-Instruct","org":"GSAI-ML","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":37,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"llada2-1-flash","name":"LLaDA2.1-flash","org":"inclusionAI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":76.59,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"llama-2-13b","name":"Llama 2-13B","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":96.28,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-07-18","release_source":"epoch","price_in":0.1,"price_out":0.5,"cheapest_provider":"replicate","cheapest_model":"meta/llama-2-13b","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-2-34b","name":"Llama 2-34B","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":93.69,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-07-18","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"llama-2-70b","name":"Llama 2-70B","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":105.64,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-07-18","release_source":"epoch","price_in":0.65,"price_out":2.75,"cheapest_provider":"replicate","cheapest_model":"meta/llama-2-70b","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-2-7b","name":"Llama 2-7B","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":82.8,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-07-18","release_source":"epoch","price_in":0.05,"price_out":0.25,"cheapest_provider":"replicate","cheapest_model":"meta/llama-2-7b","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-3-70b","name":"Llama 3-70B","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":121.34,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-04-18","release_source":"epoch","price_in":0.59,"price_out":0.79,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"meta/llama-3-70b","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-3-8b","name":"Llama 3-8B","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":109,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-04-18","release_source":"epoch","price_in":0.05,"price_out":0.08,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"meta/llama-3-8b","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-3-2-1b","name":"Llama 3.2 1B","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":11.95,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2024-09-24","release_source":"epoch","price_in":0.1,"price_out":0.1,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"meta/llama-3.2-1b","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-3-2-90b","name":"Llama 3.2 90B","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":122.89,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-09-24","release_source":"epoch","price_in":0.72,"price_out":0.72,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"meta/llama-3.2-90b","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-3-3-70b","name":"Llama 3.3 70B","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":126.16,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-12-06","release_source":"epoch","price_in":0.135,"price_out":0.432,"cheapest_provider":"kyma","cheapest_model":"llama-3.3-70b","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-4-maverick-instruct","name":"Llama 4 Maverick Instruct","org":"Meta","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":21.04,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-04-05","release_source":"curated","price_in":0.25,"price_out":0.95,"cheapest_provider":"replicate","cheapest_model":"meta/llama-4-maverick-instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-4-scout-instruct","name":"Llama 4 Scout Instruct","org":"Meta","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":9.06,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-04-05","release_source":"curated","price_in":0.17,"price_out":0.65,"cheapest_provider":"replicate","cheapest_model":"meta/llama-4-scout-instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-13b","name":"LLaMA-13B","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":84.95,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-02-24","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"llama-2-13b-chat","name":"Llama-2-13b-chat","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":42.24,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-07-18","release_source":"epoch","price_in":0.1,"price_out":0.5,"cheapest_provider":"replicate","cheapest_model":"meta/llama-2-13b-chat","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-2-70b-chat","name":"Llama-2-70b-chat","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":41.65,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-07-18","release_source":"epoch","price_in":0.65,"price_out":2.75,"cheapest_provider":"replicate","cheapest_model":"meta/llama-2-70b-chat","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-3-smaug-8b","name":"Llama-3-Smaug-8B","org":"abacusai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":36.93,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"llama-3-1-nemotron-70b-instruct-hf","name":"Llama-3.1-Nemotron-70B-Instruct-HF","org":"NVIDIA","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":54.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"llama-3-2-3b","name":"Llama-3.2-3B","org":"meta-llama","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":22.17,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.04,"price_out":0.08,"cheapest_provider":"llamagate","cheapest_model":"llama-3.2-3b","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-3-2-3b-instruct","name":"Llama-3.2-3B-Instruct","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":10.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-09-24","release_source":"epoch","price_in":0.02,"price_out":0.02,"cheapest_provider":"deepinfra","cheapest_model":"meta-llama/Llama-3.2-3B-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"llama-33b","name":"LLaMA-33B","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":96.37,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-02-24","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"llama-65b","name":"LLaMA-65B","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":101.38,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-02-24","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"llama-7b","name":"LLaMA-7B","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":76.96,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-02-24","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"llama3-70b-8192","name":"llama3-70b-8192","org":"Meta","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":49.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"llama3-swe-rl-70b","name":"Llama3-SWE-RL-70B","org":"Meta","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":41.2,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"llemma-7b","name":"llemma_7b","org":"EleutherAI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":23.45,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"longcat-2-0","name":"LongCat-2.0","org":"meituan-longcat","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":88.9,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-20","release_source":"openrouter","price_in":0.3,"price_out":1.2,"cheapest_provider":"opencode_go","cheapest_model":"longcat-2.0","price_observed_at":"2026-11-18T15:47Z"},{"id":"longcat-flash-chat","name":"LongCat-Flash-Chat","org":"meituan-longcat","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":82.7,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"longcat-flash-lite","name":"LongCat-Flash-Lite","org":"meituan-longcat","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":78.29,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"longcat-flash-thinking-2601","name":"LongCat-Flash-Thinking-2601","org":"meituan-longcat","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":80.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"magistral-medium","name":"Magistral Medium","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":1.5,"price_out":7.5,"cheapest_provider":"mistral","cheapest_model":"magistral-medium-latest","price_observed_at":"2026-11-18T15:47Z"},{"id":"magistral-medium-thinking","name":"Magistral Medium (Thinking)","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":1.5,"price_out":7.5,"cheapest_provider":"mistral","cheapest_model":"magistral-medium-latest","price_observed_at":"2026-11-18T15:47Z"},{"id":"magistral-small","name":"Magistral Small","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.15,"price_out":0.6,"cheapest_provider":"mistral","cheapest_model":"magistral-small-latest","price_observed_at":"2026-11-18T15:47Z"},{"id":"mammoth2-7b-plus","name":"MAmmoTH2-7B-Plus","org":"TIGER-Lab","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":40.85,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mammoth2-8b-plus","name":"MAmmoTH2-8B-Plus","org":"TIGER-Lab","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":43.35,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mammoth2-8x7b-plus","name":"MAmmoTH2-8x7B-Plus","org":"TIGER-Lab","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":50.4,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mathstral-7b-v0-1","name":"Mathstral-7B-v0.1","org":"mistralai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":42,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mbd-math-llada2-mini-16b","name":"MBD-Math-LLaDA2-mini-16B","org":"SJTU-DENG-Lab","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":91.96,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mellum2-12b-a2-5b-instruct","name":"Mellum2-12B-A2.5B-Instruct","org":"JetBrains","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":40.9,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.05,"price_out":0.1,"cheapest_provider":"wandb","cheapest_model":"JetBrains/Mellum2-12B-A2.5B-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"mellum2-12b-a2-5b-instruct-sft","name":"Mellum2-12B-A2.5B-Instruct-SFT","org":"JetBrains","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":38.9,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mellum2-12b-a2-5b-thinking","name":"Mellum2-12B-A2.5B-Thinking","org":"JetBrains","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":57.6,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mellum2-12b-a2-5b-thinking-sft","name":"Mellum2-12B-A2.5B-Thinking-SFT","org":"JetBrains","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":39.9,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mercury-2-5-high","name":"Mercury 2.5 (high)","org":"Inception Labs","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":301.65,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-09-08","release_source":"epoch","price_in":0.04,"price_out":0.15,"cheapest_provider":"openrouter","cheapest_model":"inception/mercury-2.5","price_observed_at":"2026-11-08T04:47Z"},{"id":"mercury-3-preview-high","name":"Mercury 3 Preview (High)","org":"","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":61.805557,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.1,"price_out":0.35,"cheapest_provider":"openrouter","cheapest_model":"inception/mercury-3","price_observed_at":"2026-11-08T04:47Z"},{"id":"mercury-3-preview-instant","name":"Mercury 3 Preview (Instant)","org":"","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.25,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.1,"price_out":0.35,"cheapest_provider":"openrouter","cheapest_model":"inception/mercury-3","price_observed_at":"2026-11-08T04:47Z"},{"id":"mercury-3-preview-low","name":"Mercury 3 Preview (Low)","org":"","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":40.97222,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.1,"price_out":0.35,"cheapest_provider":"openrouter","cheapest_model":"inception/mercury-3","price_observed_at":"2026-11-08T04:47Z"},{"id":"mercury-3-preview-medium","name":"Mercury 3 Preview (Medium)","org":"","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":41.805557,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.1,"price_out":0.35,"cheapest_provider":"openrouter","cheapest_model":"inception/mercury-3","price_observed_at":"2026-11-08T04:47Z"},{"id":"meta-llama-3-70b","name":"Meta-Llama-3-70B","org":"meta-llama","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":52.78,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.65,"price_out":2.75,"cheapest_provider":"replicate","cheapest_model":"meta/meta-llama-3-70b","price_observed_at":"2026-11-18T15:47Z"},{"id":"meta-llama-3-8b","name":"Meta-Llama-3-8B","org":"meta-llama","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":35.36,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.05,"price_out":0.25,"cheapest_provider":"replicate","cheapest_model":"meta/meta-llama-3-8b","price_observed_at":"2026-11-18T15:47Z"},{"id":"mimo-7b-base","name":"MiMo-7B-Base","org":"XiaomiMiMo","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":41.9,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mimo-7b-rl","name":"MiMo-7B-RL","org":"XiaomiMiMo","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":58.6,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mini-swe-agent-plus-beam-1-0","name":"mini-swe-agent + Beam 1.0","org":"","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":6.36,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mini-swe-agent-plus-ember-1","name":"mini-swe-agent + Ember-1","org":"","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":21.82,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mini-swe-agent-plus-gemini-3-7-flash","name":"mini-SWE-agent + Gemini 3.7 Flash","org":"","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":12.42,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mini-swe-agent-plus-gemini-3-8-flash","name":"mini-SWE-agent + Gemini 3.8 Flash","org":"","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":18.18,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mini-swe-agent-plus-inkling","name":"mini-swe-agent + Inkling","org":"","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":0,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mini-swe-agent-plus-inkling-small","name":"mini-swe-agent + Inkling Small","org":"","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":0.91,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mini-swe-agent-plus-inkling-small-2","name":"mini-swe-agent + Inkling Small 2","org":"","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":13.03,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mini-swe-agent-plus-mimo-v2-6-flash","name":"mini-swe-agent + MiMo V2.6 Flash","org":"","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":16.06,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mini-swe-agent-plus-mimo-v2-6-pro","name":"mini-swe-agent + MiMo V2.6 Pro","org":"","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":26.97,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mini-swe-agent-plus-minimax-m3","name":"mini-swe-agent + MiniMax M3","org":"","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":7.58,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mini-swe-agent-plus-nemotron-3-5-super-vl","name":"mini-swe-agent + Nemotron 3.5 Super VL","org":"","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":0.3,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"minicpm-v-4-7-35b-a3b","name":"MiniCPM-V-4.7-35B-A3B","org":"openbmb","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":83.45,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"minicpm5-2b","name":"MiniCPM5-2B","org":"openbmb","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":70.8,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"minimax-m1-40k","name":"MiniMax-M1-40k","org":"MiniMaxAI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":81.1,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"minimax-m1-80k","name":"MiniMax-M1-80k","org":"MiniMax","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":69.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-06-13","release_source":"epoch","price_in":0.55,"price_out":2.2,"cheapest_provider":"novita","cheapest_model":"minimaxai/minimax-m1-80k","price_observed_at":"2026-11-18T15:47Z"},{"id":"minimax-text-01","name":"MiniMax-Text-01","org":"MiniMaxAI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":75.7,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ministral-8b-instruct-2410","name":"Ministral-8B-Instruct-2410","org":"mistralai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":37.93,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mistral-7b-v0-1","name":"Mistral 7B v0.1","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":102.56,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-09-27","release_source":"epoch","price_in":0.05,"price_out":0.25,"cheapest_provider":"replicate","cheapest_model":"mistralai/mistral-7b-v0.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-large","name":"Mistral Large","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":128000,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":119.96,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-02-26","release_source":"epoch","price_in":0.7,"price_out":2.1,"cheapest_provider":"mistral","cheapest_model":"mistral-large-latest","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-large-2-2407","name":"Mistral Large 2 (2407)","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":60.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mistral-large-2-jul-2024","name":"Mistral Large 2 (Jul 2024)","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":126.42,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-07-24","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mistral-large-4","name":"Mistral Large 4","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":154.56,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-06","release_source":"epoch","price_in":0.7,"price_out":2.1,"cheapest_provider":"azure_ai","cheapest_model":"mistral-large-4-latest","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-large-4-medium","name":"Mistral Large 4 (medium)","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":11.143,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-06","release_source":"epoch","price_in":0.7,"price_out":2.1,"cheapest_provider":"azure_ai","cheapest_model":"mistral-large-4-latest","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-large-4-preview","name":"Mistral Large 4 Preview","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":152.71,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.7,"price_out":2.1,"cheapest_provider":"azure_ai","cheapest_model":"mistral-large-4-latest","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-medium-3","name":"Mistral Medium 3","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":132.81,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-05-07","release_source":"epoch","price_in":0.4,"price_out":2.0,"cheapest_provider":"vertex_ai","cheapest_model":"mistral-medium-3","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-nemo","name":"Mistral NeMo","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":113.81,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-07-18","release_source":"epoch","price_in":0.029,"price_out":0.03,"cheapest_provider":"openrouter","cheapest_model":"mistralai/mistral-nemo","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-7b-instruct-v0-2","name":"Mistral-7B-Instruct-v0.2","org":"mistralai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":30.84,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.05,"price_out":0.25,"cheapest_provider":"replicate","cheapest_model":"mistralai/mistral-7b-instruct-v0.2","price_observed_at":"2026-11-18T15:47Z"},{"id":"mistral-7b-v0-2","name":"Mistral-7B-v0.2","org":"mistral-community","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":30.43,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mistral-large-instruct-2407","name":"Mistral-Large-Instruct-2407","org":"mistralai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":65.91,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mistral-large-instruct-2411","name":"Mistral-Large-Instruct-2411","org":"mistralai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":67.94,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mistral-medium-2312","name":"mistral-medium-2312","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":53.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-12-11","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mistral-nemo-base-2407","name":"Mistral-Nemo-Base-2407","org":"mistralai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":39.77,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mistral-small-2402","name":"mistral-small-2402","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":52.96,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-02-26","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mistral-small-2409","name":"mistral-small-2409","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"dynamic tool-use benchmark","label":"DTBench","trust":2,"value":49.53,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"dtbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-09-17","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mistral-small-24b-base-2501","name":"Mistral-Small-24B-Base-2501","org":"mistralai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":54.4,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mistral-small-4-119b-2603","name":"Mistral-Small-4-119B-2603","org":"mistralai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":71.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mistral-small-instruct-2409","name":"Mistral-Small-Instruct-2409","org":"mistralai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":48.4,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mixtral-8x22b","name":"Mixtral 8x22B","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":118.66,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-04-17","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mixtral-8x7b","name":"Mixtral 8x7B","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":113.1,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-12-11","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mixtral-8x7b-instruct-v0-1","name":"Mixtral-8x7B-Instruct-v0.1","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":30.587,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-12-11","release_source":"epoch","price_in":0.15,"price_out":0.15,"cheapest_provider":"anyscale","cheapest_model":"mistralai/Mixtral-8x7B-Instruct-v0.1","price_observed_at":"2026-11-18T15:47Z"},{"id":"mixtral-8x7b-v0-1","name":"Mixtral-8x7B-v0.1","org":"mistralai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":41.03,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"moderato-v1-pro","name":"Moderato-V1-Pro","org":"nitrai-research","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":90,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"motif-3-base","name":"Motif-3-Base","org":"Motif-Technologies","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":93.93,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mpt-30b","name":"MPT-30B","org":"MosaicML","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":84.95,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-06-22","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"mpt-7b","name":"MPT-7B","org":"MosaicML","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":73.18,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-05-05","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"muse-code-plus-muse-ember","name":"Muse Code + Muse Ember","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":30.61,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"muse-code-plus-muse-spark-1-3","name":"Muse Code + Muse Spark 1.3","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":14.85,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"muse-code-plus-muse-wonder","name":"Muse Code + Muse Wonder","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real terminal tasks, agent+model config","label":"Terminal-Bench 4.0","trust":4,"value":46.97,"score_max":100,"source_url":"https://www.tbench.ai/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"tbench","source_label":"Terminal-Bench 4.0 official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"muse-ember-max","name":"Muse Ember (max)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":49.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-11-09","release_source":"epoch","price_in":0.1,"price_out":0.5,"cheapest_provider":"openrouter","cheapest_model":"meta/muse-ember","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark-1-1-medium","name":"Muse Spark 1.1 (medium)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":15,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-09","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"empiriolabs","cheapest_model":"muse-spark-1-1","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark-1-3-high","name":"Muse Spark 1.3 (high)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":33.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-09-02","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"empiriolabs","cheapest_model":"muse-spark-1-3","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark-1-3-low","name":"Muse Spark 1.3 (low)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":29.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-09-02","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"empiriolabs","cheapest_model":"muse-spark-1-3","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark-1-3-medium","name":"Muse Spark 1.3 (medium)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":32.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-09-02","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"empiriolabs","cheapest_model":"muse-spark-1-3","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark-1-3-minimal","name":"Muse Spark 1.3 (minimal)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":24.3,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-09-02","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"empiriolabs","cheapest_model":"muse-spark-1-3","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark-open","name":"Muse Spark Open","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":155.45,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-15","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"aiand","cheapest_model":"meta/muse-spark-open","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark-open-high","name":"Muse Spark Open (High)","org":"","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":47.5,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-15","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"aiand","cheapest_model":"meta/muse-spark-open","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark-open-low","name":"Muse Spark Open (Low)","org":"","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":12.222222,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-15","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"aiand","cheapest_model":"meta/muse-spark-open","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark-open-medium","name":"Muse Spark Open (Medium)","org":"","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":27.5,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-15","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"aiand","cheapest_model":"meta/muse-spark-open","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark-open-unknown","name":"Muse Spark Open (unknown)","org":"Meta AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long professional agent tasks","label":"APEX-Agents","trust":2,"value":33.4,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"apex_agents","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-15","release_source":"epoch","price_in":1.25,"price_out":4.25,"cheapest_provider":"aiand","cheapest_model":"meta/muse-spark-open","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-wonder","name":"Muse Wonder","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":161.74,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-12","release_source":"epoch","price_in":4.0,"price_out":20.0,"cheapest_provider":"openrouter","cheapest_model":"meta/muse-wonder","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-wonder-high","name":"Muse Wonder (high)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":46.1,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-12","release_source":"epoch","price_in":4.0,"price_out":20.0,"cheapest_provider":"openrouter","cheapest_model":"meta/muse-wonder","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-wonder-low","name":"Muse Wonder (low)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":41.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-12","release_source":"epoch","price_in":4.0,"price_out":20.0,"cheapest_provider":"openrouter","cheapest_model":"meta/muse-wonder","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-wonder-medium","name":"Muse Wonder (medium)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":45.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-12","release_source":"epoch","price_in":4.0,"price_out":20.0,"cheapest_provider":"openrouter","cheapest_model":"meta/muse-wonder","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-wonder-minimal","name":"Muse Wonder (minimal)","org":"Meta AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real-world coding-agent tasks","label":"CursorBench","trust":2,"value":34.2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"cursorbench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-12","release_source":"epoch","price_in":4.0,"price_out":20.0,"cheapest_provider":"openrouter","cheapest_model":"meta/muse-wonder","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-glimmer-30b","name":"Muse-Glimmer-30B","org":"meta-models","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":83.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-08-09","release_source":"openrouter","price_in":0.2,"price_out":0.8,"cheapest_provider":"empiriolabs","cheapest_model":"muse-glimmer-30b","price_observed_at":"2026-11-18T15:47Z"},{"id":"muse-spark-open-816b-a42b","name":"Muse-Spark-Open-816B-A42B","org":"meta-models","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":89,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":1.25,"price_out":4.25,"cheapest_provider":"together_ai","cheapest_model":"meta-models/Muse-Spark-Open-816B-A42B","price_observed_at":"2026-11-18T15:47Z"},{"id":"nemotron-3-5-super-vl-high","name":"Nemotron 3.5 Super VL (High)","org":"NVIDIA","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.5277778,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-15","release_source":"openrouter","price_in":0.081,"price_out":0.4725,"cheapest_provider":"kyma","cheapest_model":"nemotron-3.5-super-vl","price_observed_at":"2026-11-18T15:47Z"},{"id":"nemotron-3-5-super-vl-medium","name":"Nemotron 3.5 Super VL (Medium)","org":"NVIDIA","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.9444444,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-15","release_source":"openrouter","price_in":0.081,"price_out":0.4725,"cheapest_provider":"kyma","cheapest_model":"nemotron-3.5-super-vl","price_observed_at":"2026-11-18T15:47Z"},{"id":"nemotron-3-5-super-vl-none","name":"Nemotron 3.5 Super VL (None)","org":"NVIDIA","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0.8333333,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-10-15","release_source":"openrouter","price_in":0.081,"price_out":0.4725,"cheapest_provider":"kyma","cheapest_model":"nemotron-3.5-super-vl","price_observed_at":"2026-11-18T15:47Z"},{"id":"nemotron-4-15b","name":"Nemotron-4 15B","org":"Nvidia","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":98.97,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-02-26","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"nemotron-cascade-14b-thinking","name":"Nemotron-Cascade-14B-Thinking","org":"nvidia","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":69.6,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"nemotron-cascade-2-30b-a3b","name":"Nemotron-Cascade-2-30B-A3B","org":"nvidia","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":76.1,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"nemotron-h-56b-base-8k","name":"Nemotron-H-56B-Base-8K","org":"nvidia","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":60.5,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"nemotron-labs-audex-30b-a3b","name":"Nemotron-Labs-Audex-30B-A3B","org":"nvidia","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":78.9,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"nousresearch-hermes-3-llama-3-1-405b","name":"nousresearch/hermes-3-llama-3.1-405b","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":63.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":1.0,"price_out":1.0,"cheapest_provider":"deepinfra","cheapest_model":"NousResearch/Hermes-3-Llama-3.1-405B","price_observed_at":"2026-11-18T15:47Z"},{"id":"nova-2-0-pro-preview-unknown","name":"Nova 2.0 Pro Preview (unknown)","org":"Amazon","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":2,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"nova-pro","name":"Nova Pro","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":54.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.8,"price_out":3.2,"cheapest_provider":"vercel_ai_gateway","cheapest_model":"amazon/nova-pro","price_observed_at":"2026-11-18T15:47Z"},{"id":"nvidia-nemotron-3-nano-30b-a3b-base-bf16","name":"NVIDIA-Nemotron-3-Nano-30B-A3B-Base-BF16","org":"nvidia","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":65.1,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"nvidia-nemotron-3-super-120b-a12b-fp8","name":"NVIDIA-Nemotron-3-Super-120B-A12B-FP8","org":"nvidia","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":77.27273,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"o1-2024-12-17","name":"o1-2024-12-17","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":200000,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":61.7,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2024-12-17","release_source":"name","price_in":15.0,"price_out":60.0,"cheapest_provider":"azure","cheapest_model":"o1-2024-12-17","price_observed_at":"2026-11-18T15:47Z"},{"id":"o3-unknown-thinking","name":"o3 (unknown thinking)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"length of task (minutes) done at 50% success","label":"METR time horizon","trust":2,"value":119.733,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"metr_horizon","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":2.0,"price_out":8.0,"cheapest_provider":"azure","cheapest_model":"o3","price_observed_at":"2026-11-18T15:47Z"},{"id":"o3-plus-gpt-4-1","name":"o3 + gpt-4.1","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":78.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"o3-mini-low","name":"o3-mini (Low)","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":0,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2025-01-31","release_source":"curated","price_in":1.1,"price_out":4.4,"cheapest_provider":"azure","cheapest_model":"o3-mini","price_observed_at":"2026-11-18T15:47Z"},{"id":"o3-mini-1-0","name":"o3-mini_1.0","org":"OpenAI","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Lite","trust":2,"value":30.33,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_lite","source_label":"SWE-bench Lite leaderboard (unverified)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"o3-search","name":"o3-search","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1144,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ollama-codegeex4","name":"ollama/codegeex4","org":"","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":32.3,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ollama-codestral","name":"ollama/codestral","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":45.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.2,"price_out":0.6,"cheapest_provider":"vertex_ai","cheapest_model":"codestral@latest","price_observed_at":"2026-11-18T15:47Z"},{"id":"ollama-granite3-dense-8b","name":"ollama/granite3-dense:8b","org":"IBM","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":20.3,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ollama-hermes3","name":"ollama/hermes3","org":"","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":22.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ollama-hermes3-8b-llama3-1-fp16","name":"ollama/hermes3:8b-llama3.1-fp16","org":"","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":30.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ollama-llama3-2-3b-instruct-fp16","name":"ollama/llama3.2:3b-instruct-fp16","org":"Meta","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":26.3,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ollama-mistral-nemo-12b-instruct-2407-q4-k-m","name":"ollama/mistral-nemo:12b-instruct-2407-q4_K_M","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":33.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ollama-mistral-small","name":"ollama/mistral-small","org":"Mistral AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":38.3,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.1,"price_out":0.3,"cheapest_provider":"mistral","cheapest_model":"mistral-small","price_observed_at":"2026-11-18T15:47Z"},{"id":"ollama-qwen2-5-coder-14b","name":"ollama/qwen2.5-coder:14b","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":61.7,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2024-09-18","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ollama-qwen2-5-coder-32b","name":"ollama/qwen2.5-coder:32b","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":72.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2024-11-12","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ollama-qwen2-5-1-coder-7b-instruct-gguf-q8-0-32k","name":"ollama/Qwen2.5.1-Coder-7B-Instruct-GGUF:Q8_0-32k","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":63.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ollama-qwen2-5-32b","name":"ollama/qwen2.5:32b","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":54.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2024-09-17","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ollama-qwen2-5-32b-instruct-q8-0","name":"ollama/qwen2.5:32b-instruct-q8_0","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":58.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ollama-tulu3","name":"ollama/tulu3","org":"","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":26.3,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ollama-wojtek-opencodeinterpreter-6-7b","name":"ollama/wojtek/opencodeinterpreter:6.7b","org":"","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":30.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ollama-yi-coder-9b-chat-fp16","name":"ollama/yi-coder:9b-chat-fp16","org":"","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":43.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"open-mistral-nemo-2407","name":"open-mistral-nemo-2407","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":29.893,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-07-18","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"open-mixtral-8x22b","name":"open-mixtral-8x22b","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":34.059,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-04-17","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"open-llama-7b","name":"open_llama_7b","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":63.91,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-06-07","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"openai-chatgpt-4o-latest","name":"openai/chatgpt-4o-latest","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":72.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"openhands-lm-32b-v0-1","name":"openhands-lm-32b-v0.1","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":10.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"openpangu-2-0-flash","name":"openPangu-2.0-Flash","org":"openpangu","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":83.7,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"openrouter-qwen-qwen-2-5-coder-32b-instruct","name":"openrouter/qwen/qwen-2.5-coder-32b-instruct","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":65.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"optimus-alpha","name":"Optimus Alpha","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":52.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"opus-4-5-thinking-16k","name":"Opus 4.5 (Thinking, 16K)","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":22.78,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"opus-4-5-thinking-64k","name":"Opus 4.5 (Thinking, 64K)","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":37.64,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"opus-4-5-thinking-8k","name":"Opus 4.5 (Thinking, 8K)","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":13.89,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"opus-4-5-thinking-none","name":"Opus 4.5 (Thinking, None)","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":7.78,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ornith-1-5-35b-a3b","name":"Ornith-1.5-35B-A3B","org":"ornith-ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":89.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ornith-1-5-397b","name":"Ornith-1.5-397B","org":"ornith-ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":92.8,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ornith-1-5-9b","name":"Ornith-1.5-9B","org":"ornith-ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":86.4,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"palm-2-l","name":"PaLM 2-L","org":"Google","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":113.71,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-05-17","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"palm-2-m","name":"PaLM 2-M","org":"Google","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":97.77,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-05-17","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"palm-2-s","name":"PaLM 2-S","org":"Google","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":93.62,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-05-17","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"parasail-qwen3-235b-a22b-instruct-2507","name":"parasail-qwen3-235b-a22b-instruct-2507","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":52.9,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-09-01","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"phi-1-5","name":"Phi-1.5","org":"Microsoft","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":68.17,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-09-11","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"phi-2","name":"Phi-2","org":"Microsoft","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":95.94,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-12-12","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"phi-3-medium-14b","name":"phi-3-medium 14B","org":"Microsoft","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":114.01,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-04-23","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"phi-3-mini-3-8b","name":"phi-3-mini 3.8B","org":"Microsoft","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":110.4,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-04-23","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"phi-3-mini-128k-instruct","name":"Phi-3-mini-128k-instruct","org":"microsoft","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":43.86,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"phi-3-small-7-4b","name":"phi-3-small 7.4B","org":"Microsoft","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":116.26,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-04-23","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"pixtral-large-2411","name":"pixtral-large-2411","org":"Mistral AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"multi-step puzzle hunts","label":"EnigmaEval","trust":2,"value":0.84,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"enigma_eval","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-11-18","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ppl-sonar-pro-high","name":"ppl-sonar-pro-high","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1130,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"ppl-sonar-reasoning-pro-high","name":"ppl-sonar-reasoning-pro-high","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted web-search answers","label":"LMArena Search Elo","trust":2,"value":1139,"score_max":2000,"source_url":"https://arena.ai/leaderboard/search","recorded_at":"2026-10-15","source_kind":"dataset","benchmark_id":"lmarena_search","source_label":"LMArena (third-party snapshot 2026-10-07)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"quasar-alpha","name":"Quasar Alpha","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":54.7,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen-3-5-flash-hosted-35b-a3b","name":"Qwen 3.5 Flash (hosted 35B-A3B)","org":"Alibaba","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":143.87,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-02-25","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen-3-5-plus-hosted-397b-a17b","name":"Qwen 3.5 Plus (hosted 397B-A17B)","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":147.3,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-02-16","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen-3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":144.11,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-04-14","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen-3-6-flash","name":"Qwen 3.6 Flash","org":"Alibaba","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":143.76,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-04-27","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen-3-6-plus","name":"Qwen 3.6 Plus","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":148.91,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-03-31","release_source":"epoch","price_in":0.4388,"price_out":2.633,"cheapest_provider":"kyma","cheapest_model":"qwen-3.6-plus","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen-3-8-27b","name":"Qwen 3.8 27B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":152.65,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-08-14","release_source":"epoch","price_in":0.99,"price_out":1.49,"cheapest_provider":"cerebras","cheapest_model":"qwen-3.8-27b","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen-3-8-max","name":"Qwen 3.8 Max","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":157.59,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-08-02","release_source":"epoch","price_in":2.228,"price_out":6.684,"cheapest_provider":"kyma","cheapest_model":"qwen-3.8-max","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen-14b","name":"Qwen-14B","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":101.61,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-09-24","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen-1-8b","name":"Qwen-1_8B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":59.23,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-11-30","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen-2-5-72b-instruct","name":"qwen-2.5-72b-instruct","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":65.4,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.38,"price_out":0.4,"cheapest_provider":"novita","cheapest_model":"qwen/qwen-2.5-72b-instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen-7b","name":"Qwen-7B","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":88.87,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-09-28","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen1-5-110b","name":"Qwen1.5-110B","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":49.93,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen1-5-110b-chat","name":"qwen1.5-110b-chat","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":37.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen1-5-14b-chat","name":"Qwen1.5-14B-Chat","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":38.02,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen1-5-32b-chat","name":"qwen1.5-32b-chat","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":30.745,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-04-03","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen1-5-72b-chat","name":"qwen1.5-72b-chat","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":28.819,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-02-04","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen1-5-7b-chat","name":"Qwen1.5-7B-Chat","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":29.06,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-0-5b","name":"Qwen2-0.5B","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":14.97,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-1-5b","name":"Qwen2-1.5B","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":22.56,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-1-5b-instruct","name":"Qwen2-1.5B-Instruct","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":22.62,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.02,"price_out":0.02,"cheapest_provider":"together_ai","cheapest_model":"Qwen/Qwen2-1.5B-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen2-vl-72b-instruct","name":"Qwen2-VL-72B-Instruct","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":12.8,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-08-29","release_source":"epoch","price_in":0.13,"price_out":0.4,"cheapest_provider":"nebius","cheapest_model":"Qwen/Qwen2-VL-72B-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen2-vl-7b-instruct","name":"Qwen2-VL-7B-Instruct","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long-horizon game-playing agents","label":"BALROG","trust":2,"value":3.7,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"balrog","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-08-29","release_source":"epoch","price_in":0.02,"price_out":0.06,"cheapest_provider":"nebius","cheapest_model":"Qwen/Qwen2-VL-7B-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen2-5-0-5b","name":"Qwen2.5-0.5B","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":14.92,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-1-5b","name":"Qwen2.5-1.5B","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":32.1,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-14b","name":"Qwen2.5-14B","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":63.69,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-32b","name":"Qwen2.5-32B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":69.23,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2024-09-17","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-3b","name":"Qwen2.5-3B","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":43.73,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-7b","name":"Qwen2.5-7B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":45,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2024-09-19","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-coder-1-5b","name":"Qwen2.5-Coder (1.5B)","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":89.01,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-09-18","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-coder-7b","name":"Qwen2.5-Coder (7B)","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":107.45,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-09-18","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-coder-0-5b","name":"Qwen2.5-Coder-0.5B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":67.95,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-09-18","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-coder-0-5b-instruct","name":"Qwen2.5-Coder-0.5B-Instruct","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":14.3,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-coder-1-5b-instruct","name":"Qwen2.5-Coder-1.5B-Instruct","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":31.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-coder-14b","name":"Qwen2.5-Coder-14B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":113.72,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-09-18","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-coder-14b-instruct","name":"Qwen2.5-Coder-14B-Instruct","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":69.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-coder-32b","name":"Qwen2.5-Coder-32B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":118.74,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-11-12","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-coder-3b","name":"Qwen2.5-Coder-3B","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":96.63,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-09-18","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-coder-3b-instruct","name":"Qwen2.5-Coder-3B-Instruct","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":39.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.01,"price_out":0.03,"cheapest_provider":"nscale","cheapest_model":"Qwen/Qwen2.5-Coder-3B-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen2-5-coder-7b-instruct","name":"Qwen2.5-Coder-7B-Instruct","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":57.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.01,"price_out":0.03,"cheapest_provider":"nscale","cheapest_model":"Qwen/Qwen2.5-Coder-7B-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen2-5-coder-7b-instruct-q8-0","name":"qwen2.5-coder:7b-instruct-q8_0","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":51.9,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-5-max","name":"Qwen2.5-Max","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":131.17,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-01-28","release_source":"curated","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen2-72b-instruct-q8-0","name":"qwen2:72b-instruct-q8_0","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":49.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-235b-a22b-diff-no-think-alibaba-api","name":"Qwen3 235B A22B diff, no think, Alibaba API","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":59.6,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-1-7b","name":"qwen3-1.7b","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":38.005,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-04-29","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-235b-a22b-instruct-25-07","name":"Qwen3-235b-a22b Instruct (25/07)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.25,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-235b-a22b-instruct-jul-2025","name":"Qwen3-235B-A22B-Instruct (Jul 2025)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":138.07,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-07-25","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-235b-a22b-thinking-jul-2025","name":"Qwen3-235B-A22B-Thinking (Jul 2025)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":143.27,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-07-25","release_source":"epoch","price_in":0.18,"price_out":0.54,"cheapest_provider":"deepinfra","cheapest_model":"Qwen/Qwen3-235B-A22B","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-30b-a3b-base","name":"Qwen3-30B-A3B-Base","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":61.7,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-30b-a3b-instruct-jul-2025","name":"Qwen3-30B-A3B-Instruct (Jul 2025)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":134.83,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-07-29","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-30b-a3b-thinking-jul-2025","name":"Qwen3-30B-A3B-Thinking (Jul 2025)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":138.35,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-07-30","release_source":"epoch","price_in":0.08,"price_out":0.25,"cheapest_provider":"gmi","cheapest_model":"Qwen/Qwen3-30B-A3B","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-4b","name":"qwen3-4b","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":52.336,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-04-29","release_source":"epoch","price_in":0.08,"price_out":0.24,"cheapest_provider":"nebius","cheapest_model":"Qwen/Qwen3-4B","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-4b-none","name":"qwen3-4b (none)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":43.561,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-04-29","release_source":"epoch","price_in":0.08,"price_out":0.24,"cheapest_provider":"nebius","cheapest_model":"Qwen/Qwen3-4B","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-4b-thinking-2507","name":"Qwen3-4B-Thinking-2507","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":65.8,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-coder-plus","name":"qwen3-coder-plus","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"AtCoder heuristic-optimisation contests (performance rating)","label":"ALE-Bench","trust":2,"value":456.5,"score_max":4000,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"ale_bench","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-09-23","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-max","name":"Qwen3-Max","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":142.44,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-09-24","release_source":"epoch","price_in":1.2,"price_out":6.0,"cheapest_provider":"deepinfra","cheapest_model":"Qwen/Qwen3-Max","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-next-80b-a3b-instruct","name":"Qwen3-Next-80B-A3B-Instruct","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"long-context story comprehension (16k)","label":"Fiction.liveBench","trust":2,"value":55.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"fiction_live","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-09-11","release_source":"epoch","price_in":0.09,"price_out":1.1,"cheapest_provider":"deepinfra","cheapest_model":"Qwen/Qwen3-Next-80B-A3B-Instruct","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-vl-235b-a22b-thinking","name":"Qwen3-VL-235B-A22B-Thinking","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":83.8,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":0.4,"price_out":4.0,"cheapest_provider":"dashscope","cheapest_model":"qwen3-vl-235b-a22b-thinking","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-5-0-8b","name":"qwen3.5-0.8b","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-02-24","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-5-0-8b-none","name":"qwen3.5-0.8b (none)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-02-24","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-5-2b-none","name":"qwen3.5-2b (none)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"unpublished research-level physics problems","label":"CritPt","trust":2,"value":0,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"critpt","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-02-24","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-7-max-none","name":"qwen3.7-max (none)","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"deductive reasoning puzzles","label":"Mystery Games","trust":2,"value":26,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mystery_games","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-05-19","release_source":"epoch","price_in":1.25,"price_out":3.75,"cheapest_provider":"novita","cheapest_model":"qwen/qwen3.7-max","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-7-max-20260517","name":"qwen3.7-max-20260517","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1515,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true}],"n_cells":1,"released_at":"2026-05-17","release_source":"name","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwen3-7-max-preview","name":"qwen3.7-max-preview","org":"Alibaba","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"head-to-head chat preference (crowd votes)","label":"LMArena Text Elo","trust":2,"value":1475,"score_max":2000,"source_url":"https://arena.ai/leaderboard/text","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_elo","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":1.25,"price_out":3.75,"cheapest_provider":"novita","cheapest_model":"qwen/qwen3.7-max","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-8-flash-xhigh","name":"Qwen3.8 Flash (xhigh)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"economically valuable document tasks","label":"GDP.pdf","trust":2,"value":16.6,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-17","source_kind":"dataset","benchmark_id":"gdp_pdf","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-08-26","release_source":"epoch","price_in":0.1126,"price_out":0.380025,"cheapest_provider":"aihubmix","cheapest_model":"qwen3.8-flash","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-8-2-4t-a95b","name":"Qwen3.8-2.4T-A95B","org":"Qwen","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":92.6,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-08-12","release_source":"openrouter","price_in":2.0,"price_out":6.0,"cheapest_provider":"aihubmix","cheapest_model":"qwen3.8-2.4t-a95b","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwen3-8-27b-none","name":"Qwen3.8-27B (None)","org":"Alibaba","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":1.5277778,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2026-08-14","release_source":"epoch","price_in":0.06,"price_out":2.2,"cheapest_provider":"openrouter","cheapest_model":"qwen/qwen3.8-27b","price_observed_at":"2026-11-18T15:47Z"},{"id":"qwq-32b-plus-qwen-2-5-coder-instruct","name":"QwQ-32B + Qwen 2.5 Coder Instruct","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"edits across 6 languages (225 exercises)","label":"Aider Polyglot","trust":4,"value":26.2,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_polyglot","source_label":"Aider polyglot leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"qwq-plus","name":"qwq-plus","org":"Alibaba","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":65.404,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2025-04-08","release_source":"epoch","price_in":0.8,"price_out":2.4,"cheapest_provider":"dashscope","cheapest_model":"qwq-plus","price_observed_at":"2026-11-18T15:47Z"},{"id":"raon-speech-9b","name":"Raon-Speech-9B","org":"KRAFTON","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":64.05,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"redpajama-incite-7b-base","name":"RedPajama-INCITE-7B-Base","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":57.17,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-05-04","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"reflection-70b","name":"Reflection-70B","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":42.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"seed-2-1-pro-preview","name":"seed-2.1-pro-preview","org":"ByteDance","color":"#64748b","category":"preview","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"crowd-voted coding answers","label":"LMArena Code Elo","trust":2,"value":1517,"score_max":2000,"source_url":"https://arena.ai/leaderboard/code","recorded_at":"2026-10-06","source_kind":"dataset","benchmark_id":"lmarena_code","source_label":"LMArena (third-party snapshot 2026-10-05)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"seed-oss-36b-base","name":"Seed-OSS-36B-Base","org":"ByteDance-Seed","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":65.1,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"seed-oss-36b-base-wosyn","name":"Seed-OSS-36B-Base-woSyn","org":"ByteDance-Seed","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":60.4,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"sensenova-u1-a3b-mot","name":"SenseNova-U1-A3B-MoT","org":"sensenova","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":84.04,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"sensenova-u1-5-8b-mot","name":"SenseNova-U1.5-8B-MoT","org":"sensenova","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":86.67,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"smollm2-1-7b","name":"SmolLM2-1.7B","org":"HuggingFaceTB","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":18.31,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"smollm2-135m","name":"SmolLM2-135M","org":"HuggingFaceTB","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":10.85,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"smollm2-360m","name":"SmolLM2-360M","org":"HuggingFaceTB","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":11.38,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"stable-beluga-2","name":"Stable Beluga 2","org":"Stability AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":109.29,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-07-20","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"starcoder-2-15b","name":"StarCoder 2 15B","org":"Hugging Face,ServiceNow,Nvidia,BigCode","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":96.04,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-02-20","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"starcoder-2-3b","name":"StarCoder 2 3B","org":"Hugging Face,ServiceNow,Nvidia,BigCode","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":66.21,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-02-22","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"starcoder-2-7b","name":"StarCoder 2 7B","org":"Hugging Face,ServiceNow,Nvidia,BigCode","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":72.14,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-02-20","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"step-5-preview","name":"Step 5 Preview","org":"","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"crowd-voted image understanding","label":"LMArena Vision Elo","trust":2,"value":1260,"score_max":2000,"source_url":"https://arena.ai/leaderboard/vision","recorded_at":"2026-10-05","source_kind":"dataset","benchmark_id":"lmarena_vision","source_label":"LMArena (third-party snapshot 2026-10-04)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":1.0,"price_out":2.7,"cheapest_provider":"empiriolabs","cheapest_model":"step-5-preview","price_observed_at":"2026-11-18T15:47Z"},{"id":"step-3-5-flash-base","name":"Step-3.5-Flash-Base","org":"stepfun-ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":88.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"sumi-7b","name":"sumi-7b","org":"tohoku-nlp","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"grade-school math word problems","label":"GSM8K","trust":2,"value":32.8,"score_max":100,"source_url":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gsm8k","source_label":"HuggingFace GSM8K leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"swe-1-6","name":"SWE-1.6","org":"Cognition","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":9.39,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-04-07","release_source":"epoch","price_in":0.5,"price_out":2.5,"cheapest_provider":"cognition","cheapest_model":"swe-1.6","price_observed_at":"2026-11-18T15:47Z"},{"id":"swe-1-7","name":"SWE-1.7","org":"Cognition","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":41.99,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-07-08","release_source":"epoch","price_in":0.5,"price_out":2.5,"cheapest_provider":"cognition","cheapest_model":"swe-1.7","price_observed_at":"2026-11-18T15:47Z"},{"id":"swe-2-max","name":"SWE-2 (max)","org":"Cognition","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"hard programming tasks","label":"FrontierCode","trust":2,"value":50,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"frontiercode","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2026-09-10","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"swe-agent-lm-32b","name":"SWE-agent-LM-32B","org":"Qwen","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":40.2,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"swe-search","name":"swe-search","org":"Anthropic","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":62.2,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"tencent-hunyuan-large","name":"Tencent-Hunyuan-Large","org":"tencent","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":60.2,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"tiny-recursion-model-trm","name":"Tiny Recursion Model (TRM)","org":"","color":"#64748b","category":"flash","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"abstract reasoning, ARC Prize public eval set","label":"ARC-AGI-2","trust":4,"value":6.25,"score_max":100,"source_url":"https://arcprize.org/leaderboard","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"arc_agi_2","source_label":"ARC Prize official leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"tts-bo16","name":"TTS(Bo16)","org":"OpenAI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":2,"value":58.8,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard (unverified)","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"tts-bo8","name":"TTS(Bo8)","org":"Qwen","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"real GitHub issues resolved","label":"SWE-bench Verified","trust":4,"value":47,"score_max":100,"source_url":"https://www.swebench.com","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"swe_verified","source_label":"SWE-bench Verified leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"vicuna-13b-v1-1","name":"vicuna-13b-v1.1","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":75.78,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-04-12","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"xgen-7b","name":"XGen-7B","org":"Salesforce","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":69.95,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-06-27","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"yi-coder-9b-chat","name":"Yi Coder 9B Chat","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":54.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"yi-1-5-6b-chat","name":"Yi-1.5-6B-Chat","org":"01-ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":38.23,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"yi-1-5-9b-chat","name":"Yi-1.5-9B-Chat","org":"01-ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":45.95,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"yi-34b","name":"Yi-34B","org":"01.AI","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":43.03,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":"2023-11-02","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"yi-34b-chat","name":"Yi-34B-Chat","org":"01.AI","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":14.741,"score_max":100,"source_url":"https://epoch.ai/benchmarks","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2023-11-22","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"yi-6b-chat","name":"Yi-6B-Chat","org":"01-ai","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"57-subject knowledge + reasoning","label":"MMLU-Pro","trust":2,"value":28.84,"score_max":100,"source_url":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"mmlu_pro","source_label":"HuggingFace MMLU-Pro leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"yi-9b","name":"Yi-9B","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Epoch Capabilities Index: Epoch's own aggregate of ~40 benchmarks","label":"Epoch ECI","trust":2,"value":102.06,"score_max":250,"source_url":"https://epoch.ai/benchmarks/eci","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"epoch_eci","source_label":"Epoch AI Benchmarking Hub (CC BY 4.0)","higher_is_better":true}],"n_cells":1,"released_at":"2024-03-01","release_source":"epoch","price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"yi-coder-9b-chat-q4-0","name":"yi-coder:9b-chat-q4_0","org":"","color":"#64748b","category":"frontier","params_b":null,"context_window":null,"open_weight":false,"cells":[{"note":"Aider official benchmark","label":"Aider Code Editing","trust":4,"value":45.1,"score_max":100,"source_url":"https://aider.chat/docs/leaderboards/edit.html","recorded_at":"2026-11-18","source_kind":"official","benchmark_id":"aider_edit","source_label":"Aider Aider Code Editing leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null},{"id":"zgcm-1-7b","name":"ZGCM-1-7B","org":"zgcagi","color":"#64748b","category":"local","params_b":null,"context_window":null,"open_weight":true,"cells":[{"note":"expert-level science questions","label":"GPQA Diamond","trust":2,"value":47.87,"score_max":100,"source_url":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard","recorded_at":"2026-11-18","source_kind":"dataset","benchmark_id":"gpqa_diamond","source_label":"HuggingFace GPQA Diamond leaderboard","higher_is_better":true}],"n_cells":1,"released_at":null,"release_source":null,"price_in":null,"price_out":null,"cheapest_provider":null,"cheapest_model":null,"price_observed_at":null}]