{"generated_utc":"2026-09-28T00:39:36+00:00","suite":{"requests":120340,"scoreable":119898,"benchmarks":43,"jev_benchmarks":42,"panel":38,"panel_full":38,"categories":5,"dropped":["MiniWoB++","Boxoban","RTFM","ScienceWorld","Hanabi","Codenames"],"hardware":"1 x NVIDIA RTX PRO 6000","jev_version":"jev-1.13.0","panel_id":"decision-index-0.2.1","calibration_note":"32 benchmarks with a per-field right or wrong answer (ToolRet, BRIGHT, RouterBench and ForecastBench excluded), each weighted equally; a 1-in-6 sample of requests; confidence is the probability placed on the chosen option. Rows an entrant trained on count as wrong here too.","formulas":[{"id":"balanced_raw","normalization":"raw","formula":"100*sum_c(0.2*category[c])"},{"id":"balanced_skill","normalization":"skill","formula":"100*sum_c(0.2*category[c])"},{"id":"breadth_skill","normalization":"skill","shift":0.1,"formula":"100*(product_c((0.1+0.9*category[c])**0.2)-0.1)/0.9"}],"edition":"release-v2.1","label":"Decision Index 0.2.1","corpus_sha256":"b2b56d6fb636837ca469e689087bdbf373dda8de7638aa2da6793e6eda0792d5","base_edition":"release-v2","note":null,"added":{"edition":"v0.2","benchmarks":[56,57,58,59,61,62,64],"requests":30419,"in_panel":true,"note":"Added in 0.2 and part of the index panel; every entrant on the 0.2 board has results on all of them."},"areas":{"knowledge":[25,30,31,32,33,43,44,45,57,58],"language":[11,12,28,29,38,39,40,41,42,59],"retrieval":[4,5,36,37,56,61],"tools":[1,2,3,9,62],"arts":[20,21,22,23,48,50,64]},"not_in_index":[{"id":6,"name":"RouterBench"},{"id":10,"name":"SGD"},{"id":24,"name":"MMLU"},{"id":26,"name":"ARC-Easy"},{"id":27,"name":"ARC-Challenge"}],"lower_rules":[{"id":48,"name":"ForecastBench","baseline":0.25,"best":0.0,"metric":"Brier loss","baseline_label":"always predict p = 0.5","rule":"clip((0.25 - Brier) / (0.25 - 0)) x coverage"}],"headline":"balanced_skill","chance_levels":[{"id":25,"name":"GPQA Diamond","area":"knowledge","chance":0.25,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":30,"name":"GSM8K","area":"knowledge","chance":0.175,"label":"Uniform random choice","method":"per track: 4-choice and 10-choice versions corrected separately, then averaged"},{"id":31,"name":"ChessBench","area":"knowledge","chance":0.0819,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":32,"name":"MuSR","area":"knowledge","chance":0.371,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":33,"name":"SATA-Bench","area":"knowledge","chance":0.0131,"label":"Random whole-case guesses","method":"expected score of uniform random choice over each question's options"},{"id":43,"name":"CRUXEval","area":"knowledge","chance":0.3697,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":44,"name":"CLadder","area":"knowledge","chance":0.5,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":45,"name":"HLE","area":"knowledge","chance":0.1641,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":57,"name":"MMLU-Pro","area":"knowledge","chance":0.1109,"label":"uniform random choice, mean of 1/options","method":"expected score of uniform random choice over each question's options"},{"id":58,"name":"BBH","area":"knowledge","chance":0.3101,"label":"uniform random choice, mean of 1/options","method":"expected score of uniform random choice over each question's options"},{"id":11,"name":"ContractNLI","area":"language","chance":0.3085,"label":"Random choices · F1","method":"F1 of random guessing over the fixed label set"},{"id":12,"name":"ANLI","area":"language","chance":0.3324,"label":"Random choices · F1","method":"expected score of uniform random choice over each question's options"},{"id":28,"name":"WinoGrande","area":"language","chance":0.5,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":29,"name":"HellaSwag","area":"language","chance":0.25,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":38,"name":"ACOS","area":"language","chance":0.031,"label":"Always yes · per-review F1","method":"per-review F1 of answering yes to every aspect-and-sentiment pair (a fair coin scores lower)"},{"id":39,"name":"FinEntity","area":"language","chance":0.3201,"label":"Random choices · F1","method":"expected score of uniform random choice over each question's options"},{"id":40,"name":"iSarcasmEval","area":"language","chance":0.2227,"label":"Random choices · F1","method":"track A English: F1 of random guessing on the sarcastic class"},{"id":41,"name":"VAST","area":"language","chance":0.3333,"label":"Random choices · F1","method":"F1 of random guessing over the fixed label set"},{"id":42,"name":"NLI4CT","area":"language","chance":0.486,"label":"Random choices · F1","method":"expected score of uniform random choice over each question's options"},{"id":59,"name":"RAGTruth","area":"language","chance":0.5177,"label":"Always \"hallucinated\" · F1","method":"F1 of always answering \"hallucinated\" (a fair coin scores lower)"},{"id":4,"name":"BANKING77","area":"retrieval","chance":0.0127,"label":"Random choices · F1","method":"expected score of uniform random choice over each question's options"},{"id":5,"name":"CLINC150","area":"retrieval","chance":0.006,"label":"Random choices · F1","method":"expected score of uniform random choice over each question's options"},{"id":36,"name":"BRIGHT","area":"retrieval","chance":0.116,"label":"Random ranking, answerable queries","method":"per query: expected nDCG@10 of a random ranking of that query's candidates, answerable queries only"},{"id":37,"name":"Amazon ESCI","area":"retrieval","chance":0.2027,"label":"Random choices · F1","method":"F1 of random guessing over the fixed label set"},{"id":56,"name":"PhishNChips","area":"retrieval","chance":0.5,"label":"uniform random choice, mean of 1/options","method":"expected score of uniform random choice over each question's options"},{"id":61,"name":"HoVer","area":"retrieval","chance":0.5,"label":"uniform random choice, mean of 1/options","method":"expected score of uniform random choice over each question's options"},{"id":1,"name":"BFCL","area":"tools","chance":0.2592,"label":"Random whole-case guesses","method":"all fields of a case right by chance"},{"id":2,"name":"ToolRet","area":"tools","chance":0.1341,"label":"Random ranking, answerable queries","method":"per query: expected nDCG@10 of a random ranking of that query's candidates, answerable queries only"},{"id":3,"name":"API-Bank","area":"tools","chance":0.0189,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":9,"name":"Home appliances","area":"tools","chance":0.0,"label":"Random whole-case guesses","method":"all fields of a case right by chance, on the kept rows"},{"id":62,"name":"When2Call","area":"tools","chance":0.25,"label":"uniform random choice, mean of 1/options","method":"expected score of uniform random choice over each question's options"},{"id":20,"name":"BPoMP","area":"arts","chance":0.5,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":21,"name":"Humicroedit","area":"arts","chance":0.5,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":22,"name":"POP909","area":"arts","chance":0.0078,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":23,"name":"cfcolor","area":"arts","chance":0.5,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":48,"name":"ForecastBench","area":"arts","chance":0.0,"label":"Always predict p = 0.5","method":"baseline Brier 0.25 (always predict 0.5); the baseline-converted value is already 0 there"},{"id":50,"name":"Habermas","area":"arts","chance":0.311,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":64,"name":"New Yorker","area":"arts","chance":0.2,"label":"uniform random choice, mean of 1/options","method":"expected score of uniform random choice over each question's options"}],"index_note":"Decision Index 0.2.1 averages 38 benchmarks in five areas. Arts & Human Taste weighs 10%; the other four share 90% in proportion to the square root of their benchmark count (knowledge 25.8%, language 25.8%, retrieval 20.0%, tools 18.3%). Inside an area, gold ★ benchmarks weigh 1.2 and the rest 1.0; the index is 100 x the weighted mean of the five areas. Each benchmark is chance-corrected first, (score - chance) / (1 - chance) clipped to 0-1, so 0 means random guessing and 100 means perfect. Every score is coverage-adjusted, so an unanswered or unsupported request counts as wrong. ForecastBench enters against its baseline: clip((0.25 - Brier) / 0.25) x coverage, so always predicting 0.5 scores zero. MMLU, ARC-Easy, ARC-Challenge, RouterBench, SGD stay on the board as non-index benchmarks. The six interactive environments are still unrun and stay out. Every entrant on the board has results on all 38 index benchmarks. Point estimates only, no uncertainty intervals yet.","changes":["New area weights: Arts & Human Taste is fixed at 10%; the other four areas share the rest in proportion to the square root of their benchmark count: Tools & Automation 18.3%, Retrieval & Classification 20.0%, Knowledge & Reasoning 25.8%, Language Understanding 25.8%.","Gold ★ benchmarks weigh 1.2 inside their area, the rest 1.0: MMLU-Pro, BBH, GPQA Diamond, HLE, ANLI, WinoGrande, HellaSwag, BANKING77, CLINC150, BRIGHT, BFCL, API-Bank, ForecastBench.","SGD leaves the index until it is re-run with a fixed builder, and RouterBench leaves because its prompt gives away the best route. Both stay on the board.","ACOS is now scored with F1 per review instead of all-or-nothing, and stays in the index.","ToolRet and BRIGHT count only queries with at least one relevant candidate among the 32 (685 of 1,000 and 220 of 550); chance is recomputed on the same queries.","RAGTruth chance is now always answering \"hallucinated\" (F1 0.518) instead of a fair coin (0.411).","Home appliances drops 24 duplicate test rows and 48 rows identical to published dev rows (88 of 160 kept).","Surogate Rune 26B-A4B v3 replaces v1 on the board: the author's submitted run, adopted after our 3,000-row reproduction matched it exactly.","reflex 27B's full-coverage rerun replaces its 26-option run on the board: the author's submitted run with wide-choice labels, adopted after our single-card latency run agreed with it within reflex's own run-to-run noise.","Hopper (G) 1.2 replaces Hopper 1.1.1 on the board: the author's submitted run of the next Hopper adapter (same base and serving code), adopted after our single-card latency run agreed with it within the A10G-vs-RTX noise."],"area_weights":{"knowledge":0.258494,"language":0.258494,"retrieval":0.200229,"tools":0.182783,"arts":0.1},"area_sizing":"sqrt","gold":{"57":1.2,"58":1.2,"25":1.2,"45":1.2,"12":1.2,"28":1.2,"29":1.2,"4":1.2,"5":1.2,"36":1.2,"1":1.2,"3":1.2,"48":1.2},"index_notes":[{"id":6,"name":"RouterBench","note":"Not in the index: the prompt reveals that GPT-4 is the best route, so the top models all pick GPT-4 and score within a fraction of a point of each other. Scores stay visible."},{"id":10,"name":"SGD","note":"Not in the index: build bug, pending re-run. The builder gave every SGD row the whole dialogue, including the turns after the one being classified. Scores stay visible; SGD returns to the index once re-run with the fixed builder."}]},"categories":[{"id":"knowledge","label":"Knowledge & Reasoning","color":"#5976bd","icon":"M4 4h6a3 3 0 0 1 3 3v14a4 4 0 0 0-4-2H4z M20 4h-4a3 3 0 0 0-3 3v14a4 4 0 0 1 4-2h3z","description":"Academic knowledge, commonsense, mathematics, and code.","panel":[25,30,31,32,33,43,44,45,57,58],"panel_full":[25,30,31,32,33,43,44,45,57,58],"suite":[57,25,26,27,30,32,33,43,44,45,24,58,31],"group":[57,25,26,27,30,32,33,43,44,45,51,52,24,58],"priority":[57,25,30,45,28]},{"id":"language","label":"Language Understanding","color":"#407e9e","icon":"M4 4h16v12H9l-5 4z M8 8h8 M8 12h5","description":"Read a text and judge what it means: entailment, sentiment, sarcasm and stance.","panel":[11,12,28,29,38,39,40,41,42,59],"panel_full":[11,12,28,29,38,39,40,41,42,59],"suite":[11,12,38,39,40,41,42,28,29,59],"group":[11,12,38,39,40,41,42,28,29,59],"priority":[40,42,39,38,11]},{"id":"retrieval","label":"Retrieval & Classification","color":"#3e8e5e","icon":"M11 4a7 7 0 1 0 0 14 7 7 0 0 0 0-14z M20 20l-4.3-4.3","description":"Pick the right item from a large set: rank documents and products by relevance, route a message to one of many intents, sort out spam and phishing.","panel":[4,5,36,37,56,61],"panel_full":[4,5,36,37,56,61],"suite":[36,37,4,5,10,56,61],"group":[36,37,4,5,10,54,55,56,61],"priority":[36,37,4,5,10,55]},{"id":"tools","label":"Tools & Automation","color":"#b77a38","icon":"M5 4v11a4 4 0 0 0 4 4h10 M5 9h9a4 4 0 0 0 4-4 M2 7l3-3 3 3 m8 9 3 3-3 3","description":"Tool choice, model routing, household control, and browser or simulator actions.","panel":[1,2,3,9,62],"panel_full":[1,2,3,9,62],"suite":[1,2,3,6,9,62],"group":[1,2,3,6,9,53,62,7,16],"priority":[1,9,6]},{"id":"arts","label":"Arts & Human Taste","color":"#b6698a","icon":"M12 3a9 9 0 1 0 0 18h2a2 2 0 0 0 0-4h-1a2 2 0 0 1 0-4h4a4 4 0 0 0 4-4c0-4-5-6-9-6 M7 8h.01 M12 6h.01 M17 8h.01 M6 13h.01","description":"Poetry, music, humor, human preferences, and forecasting.","panel":[20,21,22,23,48,50,64],"panel_full":[20,21,22,23,48,50,64],"suite":[20,21,22,23,48,50,64],"group":[20,21,22,23,48,50,64],"priority":[20,22,21,23,48]},{"id":"games","label":"Games & Strategy","color":"#ad6f48","icon":"M7 8h10a4 4 0 0 1 4 4l1 5a2 2 0 0 1-3 2l-3-3H8l-3 3a2 2 0 0 1-3-2l1-5a4 4 0 0 1 4-4z M7 10v4 M5 12h4 M16 11h.01 M18 13h.01","description":"Move selection, spatial planning, rules, and decisions against other players.","panel":[],"panel_full":[31],"suite":[],"jev_only":true,"folded_into":"knowledge","group":[31,13,15,17,19],"priority":[31]}],"benchmarks":{"1":{"id":1,"dataset":"BFCL","short":"BFCL","metric":"case exact accuracy","explainer":"Says yes or no for each supplied function: call it for this request? All must be right.","baseline":{"value":0.2592,"label":"Random whole-case guesses"},"lower":false,"range":null,"cases":1694,"group":"tools","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2305.15334"},{"kind":"code","url":"https://github.com/gorilla-llm/gorilla"}],"gold":1.2},"2":{"id":2,"dataset":"ToolRet","short":"ToolRet","metric":"nDCG@10","explainer":"Rates how useful each of 32 search-hit tools is for a request; ranked by nDCG@10, on requests where at least one of the 32 is relevant.","baseline":{"value":0.1341,"label":"Random ranking, answerable queries","previous":0.0937},"lower":false,"range":null,"cases":685,"group":"tools","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2503.01763"},{"kind":"dataset","url":"https://huggingface.co/datasets/mangopy/ToolRet-Queries"}]},"3":{"id":3,"dataset":"API-Bank","short":"API-Bank","metric":"accuracy","explainer":"Reads an assistant conversation and picks the next API to call from a catalog of 53.","baseline":{"value":0.0189,"label":"Uniform random choice"},"lower":false,"range":null,"cases":508,"group":"tools","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2304.08244"},{"kind":"code","url":"https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/api-bank"}],"gold":1.2},"4":{"id":4,"dataset":"BANKING77","short":"BANKING77","metric":"macro-F1","explainer":"Picks the intent of a bank customer's message, e.g. \"How do I locate my card?\", from 77.","baseline":{"value":0.0127,"label":"Random choices · F1"},"lower":false,"range":null,"cases":3080,"group":"retrieval","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2003.04807"},{"kind":"dataset","url":"https://github.com/PolyAI-LDN/task-specific-datasets"}],"gold":1.2},"5":{"id":5,"dataset":"CLINC150+OOS","short":"CLINC150","metric":"macro-F1","explainer":"Reads a short assistant request and picks its intent from 150, or \"out of scope\" when none fits.","baseline":{"value":0.006,"label":"Random choices · F1"},"lower":false,"range":null,"cases":5500,"group":"retrieval","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://github.com/clinc/oos-eval"}],"gold":1.2},"6":{"id":6,"dataset":"RouterBench","short":"RouterBench","metric":"selected quality (quality objective)","explainer":"Picks which of 11 AI models should answer a prompt; scored by its real answer.","baseline":{"value":0.5735,"label":"Random model selection"},"lower":false,"range":null,"cases":5001,"group":"tools","in_suite":true,"interactive":false,"index_note":"Not in the index: the prompt reveals that GPT-4 is the best route, so the top models all pick GPT-4 and score within a fraction of a point of each other. Scores stay visible.","references":[{"kind":"paper","url":"https://arxiv.org/abs/2403.12031"},{"kind":"dataset","url":"https://huggingface.co/datasets/withmartian/routerbench"}]},"9":{"id":9,"dataset":"Home appliance simulator","short":"Home appliances","metric":"case exact accuracy","explainer":"Given a simulated home and a command, answers ~18 questions on handling it; all must be right. Duplicate and dev-set rows removed.","baseline":{"value":0.0,"label":"Random whole-case guesses"},"lower":false,"range":null,"cases":88,"group":"tools","in_suite":true,"interactive":false,"references":[{"kind":"code","url":"https://github.com/apolinario/decision-index/blob/main/decision_index/suite/build/home_appliance.py"}]},"10":{"id":10,"dataset":"SGD/SGD-X","short":"SGD","metric":"macro-F1","explainer":"Reads a service dialogue so far and picks the intent the user is pursuing now, or NONE.","baseline":{"value":0.399,"label":"Random choices · F1"},"lower":false,"range":null,"cases":2500,"group":"retrieval","in_suite":true,"interactive":false,"index_note":"Not in the index: build bug, pending re-run. The builder gave every SGD row the whole dialogue, including the turns after the one being classified. Scores stay visible; SGD returns to the index once re-run with the fixed builder.","references":[{"kind":"paper","url":"https://arxiv.org/abs/1909.05855"},{"kind":"dataset","url":"https://github.com/google-research-datasets/dstc8-schema-guided-dialogue"}]},"11":{"id":11,"dataset":"ContractNLI","short":"ContractNLI","metric":"macro-F1","explainer":"Says whether a full NDA supports, contradicts or omits each of 17 fixed statements.","baseline":{"value":0.3085,"label":"Random choices · F1"},"lower":false,"range":null,"cases":123,"group":"language","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://aclanthology.org/2021.findings-emnlp.164"},{"kind":"dataset","url":"https://stanfordnlp.github.io/contract-nli/"}]},"12":{"id":12,"dataset":"ANLI","short":"ANLI","metric":"macro-F1","explainer":"Decides whether a claim follows from, contradicts, or is neutral to a premise written to fool AI models.","baseline":{"value":0.3324,"label":"Random choices · F1"},"lower":false,"range":null,"cases":3200,"group":"language","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/1910.14599"},{"kind":"dataset","url":"https://huggingface.co/datasets/facebook/anli"}],"gold":1.2},"20":{"id":20,"dataset":"BPoMP","short":"BPoMP","metric":"accuracy","explainer":"Given two versions of a limerick, one with its rhyme, meter or sense broken, picks the original.","baseline":{"value":0.5,"label":"Uniform random choice"},"lower":false,"range":null,"cases":811,"group":"arts","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://zenodo.org/records/7299879"}]},"21":{"id":21,"dataset":"Humicroedit","short":"Humicroedit","metric":"accuracy","explainer":"Given two one-word edits of a news headline, picks the one people rated funnier.","baseline":{"value":0.5,"label":"Uniform random choice"},"lower":false,"range":null,"cases":2628,"group":"arts","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/1906.00274"},{"kind":"dataset","url":"https://cs.rochester.edu/u/nhossain/humicroedit.html"}]},"22":{"id":22,"dataset":"POP909-CL","short":"POP909","metric":"accuracy","explainer":"Given the notes around a beat in a pop song (not audio), picks the chord playing from 129.","baseline":{"value":0.0078,"label":"Uniform random choice"},"lower":false,"range":null,"cases":2000,"group":"arts","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2510.06528"},{"kind":"dataset","url":"https://github.com/AndyWeasley2004/POP909-CL-Dataset"}]},"23":{"id":23,"dataset":"cfcolor","short":"cfcolor","metric":"accuracy","explainer":"Given a person's earlier ratings of color palettes, predicts which of two new palettes they rated higher.","baseline":{"value":0.5,"label":"Uniform random choice"},"lower":false,"range":null,"cases":5000,"group":"arts","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://www.dgp.toronto.edu/~donovan/cfcolor/"}]},"24":{"id":24,"dataset":"MMLU","short":"MMLU","metric":"accuracy","explainer":"Four-option questions across 57 school and professional subjects, from law to algebra.","baseline":{"value":0.25,"label":"Uniform random choice"},"lower":false,"range":null,"cases":14042,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/cais/mmlu"}]},"25":{"id":25,"dataset":"GPQA Diamond","short":"GPQA Diamond","metric":"accuracy","explainer":"198 hard four-option science questions written by PhD experts in physics, chemistry and biology.","baseline":{"value":0.25,"label":"Uniform random choice"},"lower":false,"range":null,"cases":198,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://github.com/idavidrein/gpqa"}],"gold":1.2},"26":{"id":26,"dataset":"ARC-Easy","short":"ARC-Easy","metric":"accuracy","explainer":"Grade-school science multiple choice, the easier set.","baseline":{"value":0.2502,"label":"Uniform random choice"},"lower":false,"range":null,"cases":2376,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/allenai/ai2_arc"}]},"27":{"id":27,"dataset":"ARC-Challenge","short":"ARC-Challenge","metric":"accuracy","explainer":"Grade-school science multiple choice, the harder set that simple retrieval gets wrong.","baseline":{"value":0.2502,"label":"Uniform random choice"},"lower":false,"range":null,"cases":1172,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/allenai/ai2_arc"}]},"28":{"id":28,"dataset":"WinoGrande","short":"WinoGrande","metric":"accuracy","explainer":"Fills a blank with one of two options using common sense, e.g. who \"got the easier cases\".","baseline":{"value":0.5,"label":"Uniform random choice"},"lower":false,"range":null,"cases":1267,"group":"language","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/allenai/winogrande"}],"gold":1.2},"29":{"id":29,"dataset":"HellaSwag","short":"HellaSwag","metric":"accuracy","explainer":"Picks the most sensible of four continuations of an everyday scene.","baseline":{"value":0.25,"label":"Uniform random choice"},"lower":false,"range":null,"cases":10042,"group":"language","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/Rowan/hellaswag"}],"gold":1.2},"30":{"id":30,"dataset":"GSM8K","short":"GSM8K","metric":"accuracy","explainer":"Picks a math word problem's answer from 4, and separately 10, numbers, with no working.","baseline":{"value":0.175,"label":"Uniform random choice"},"lower":false,"range":null,"cases":1319,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/openai/gsm8k"}]},"31":{"id":31,"dataset":"ChessBench","short":"ChessBench","metric":"accuracy","explainer":"Given a chess position and every legal move, picks the strongest one; tied best moves count.","baseline":{"value":0.0819,"label":"Uniform random choice"},"lower":false,"range":null,"cases":5000,"group":"games","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2402.04494"},{"kind":"code","url":"https://github.com/google-deepmind/searchless_chess"}]},"32":{"id":32,"dataset":"MuSR","short":"MuSR","metric":"accuracy","explainer":"Reads a long story and answers one question needing clues from across it, e.g. who the murderer is.","baseline":{"value":0.371,"label":"Uniform random choice"},"lower":false,"range":null,"cases":752,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2310.16049"},{"kind":"dataset","url":"https://github.com/Zayne-sprague/MuSR"}]},"33":{"id":33,"dataset":"SATA-Bench","short":"SATA-Bench","metric":"case exact accuracy","explainer":"Given a passage, a question and candidate answers, says yes or no to each; all must be right.","baseline":{"value":0.0131,"label":"Random whole-case guesses"},"lower":false,"range":null,"cases":1650,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://github.com/sata-bench/sata-bench"}]},"36":{"id":36,"dataset":"BRIGHT","short":"BRIGHT","metric":"nDCG@10","explainer":"Rates how helpful each of 32 search-hit documents is for a hard question; ranked by nDCG@10, on questions where at least one of the 32 is relevant.","baseline":{"value":0.116,"label":"Random ranking, answerable queries","previous":0.0448},"lower":false,"range":null,"cases":220,"group":"retrieval","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2407.12883"},{"kind":"dataset","url":"https://huggingface.co/datasets/xlangai/BRIGHT"}],"gold":1.2},"37":{"id":37,"dataset":"Amazon ESCI","short":"Amazon ESCI","metric":"macro-F1","explainer":"Labels a product as an exact match, substitute, complement or irrelevant to a shopping query.","baseline":{"value":0.2027,"label":"Random choices · F1"},"lower":false,"range":null,"cases":5000,"group":"retrieval","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://github.com/amazon-science/esci-data"}]},"38":{"id":38,"dataset":"ACOS","short":"ACOS","metric":"per-review F1","explainer":"Reads a restaurant or laptop review and says yes/no for each aspect-and-sentiment pair; scored by F1 per review, averaged over reviews.","baseline":{"value":0.031,"label":"Always yes · per-review F1","previous":0.0},"lower":false,"range":null,"cases":400,"group":"language","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://aclanthology.org/2021.acl-long.29"},{"kind":"dataset","url":"https://github.com/NUSTM/ACOS"}]},"39":{"id":39,"dataset":"FinEntity","short":"FinEntity","metric":"macro-F1","explainer":"Says whether a financial news snippet is positive, negative or neutral about a named firm.","baseline":{"value":0.3201,"label":"Random choices · F1"},"lower":false,"range":null,"cases":979,"group":"language","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://aclanthology.org/2023.emnlp-main.956"},{"kind":"dataset","url":"https://github.com/yixuantt/FinEntity"}]},"40":{"id":40,"dataset":"iSarcasmEval","short":"iSarcasmEval","metric":"Sarcasm F1 · track A, English","explainer":"Says yes or no: is this tweet sarcastic? Scored with F1 on sarcastic English tweets (track A); other tracks listed under each result.","baseline":{"value":0.2227,"label":"Random choices · F1"},"lower":false,"range":null,"cases":4600,"group":"language","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://aclanthology.org/2022.semeval-1.111"},{"kind":"dataset","url":"https://github.com/iabufarha/iSarcasmEval"}]},"41":{"id":41,"dataset":"VAST","short":"VAST","metric":"macro-F1","explainer":"Says whether a comment is for, against, or neutral on a given topic.","baseline":{"value":0.3333,"label":"Random choices · F1"},"lower":false,"range":null,"cases":3006,"group":"language","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://github.com/emilyallaway/zero-shot-stance"}]},"42":{"id":42,"dataset":"NLI4CT","short":"NLI4CT","metric":"macro-F1","explainer":"Says whether clinical-trial report sections support or contradict a statement, some deliberately altered.","baseline":{"value":0.486,"label":"Random choices · F1"},"lower":false,"range":null,"cases":5500,"group":"language","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://github.com/ai-systems/Task-2-SemEval-2024"}]},"43":{"id":43,"dataset":"CRUXEval","short":"CRUXEval","metric":"accuracy","explainer":"Reads a short Python function and an input and picks its output from 2 to 10 candidates, without running it.","baseline":{"value":0.3697,"label":"Uniform random choice"},"lower":false,"range":null,"cases":570,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2401.03065"},{"kind":"dataset","url":"https://github.com/facebookresearch/cruxeval"}]},"44":{"id":44,"dataset":"CLadder","short":"CLadder","metric":"accuracy","explainer":"Answers a yes/no question about cause and effect in a small made-up world with stated probabilities.","baseline":{"value":0.5,"label":"Uniform random choice"},"lower":false,"range":null,"cases":5000,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2312.04350"},{"kind":"dataset","url":"https://github.com/causalNLP/cladder"}]},"45":{"id":45,"dataset":"HLE","short":"HLE","metric":"accuracy","explainer":"Expert-level multiple choice from Humanity's Last Exam, text-only questions, 5 to 26 options.","baseline":{"value":0.1641,"label":"Uniform random choice"},"lower":false,"range":null,"cases":501,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/cais/hle"}],"gold":1.2},"48":{"id":48,"dataset":"ForecastBench","short":"ForecastBench","metric":"Brier (lower is better)","explainer":"Gives the probability that an event resolving July–September 2026 happens, from earlier information only.","baseline":{"value":0.25,"label":"Always predict p = 0.5"},"lower":true,"range":{"best":0.0,"worst":1.0,"name":"Brier loss"},"cases":10139,"group":"arts","in_suite":true,"interactive":false,"index_rule":{"mode":"vs baseline","baseline":0.25,"best":0.0,"rule":"clip((0.25 - Brier) / (0.25 - 0)) x coverage","baseline_label":"always predict p = 0.5"},"references":[{"kind":"paper","url":"https://arxiv.org/abs/2409.19839"},{"kind":"dataset","url":"https://github.com/forecastingresearch/forecastbench-datasets"}],"gold":1.2},"50":{"id":50,"dataset":"Habermas Machine","short":"Habermas","metric":"accuracy","explainer":"Given a group's opinions on a policy question, predicts which candidate statement the group ranked highest.","baseline":{"value":0.311,"label":"Uniform random choice"},"lower":false,"range":null,"cases":1676,"group":"arts","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://github.com/google-deepmind/habermas_machine"}]},"51":{"id":51,"dataset":"OpenBookQA","short":"OpenBookQA","metric":"accuracy","explainer":"Answer elementary-science questions from four original choices, without the supporting fact.","baseline":{"value":0.25,"label":"Uniform random choice"},"lower":false,"range":null,"cases":500,"group":"knowledge","in_suite":false,"interactive":false},"52":{"id":52,"dataset":"CommonsenseQA","short":"CommonsenseQA","metric":"accuracy","explainer":"Answer everyday commonsense questions from five original choices.","baseline":{"value":0.2,"label":"Uniform random choice"},"lower":false,"range":null,"cases":1221,"group":"knowledge","in_suite":false,"interactive":false},"53":{"id":53,"dataset":"Support-ticket calibration","short":"Support tickets","metric":"accuracy","explainer":"Judge a support ticket for queue, anger and priority, against deliberately noisy labels.","baseline":{"value":null,"label":null},"lower":false,"range":null,"cases":300,"group":"tools","in_suite":false,"interactive":false},"54":{"id":54,"dataset":"Phishing difficulty gradient","short":"Phishing gradient","metric":"accuracy","explainer":"Judge generated emails as phishing across four difficulty tiers, from obvious to misleading.","baseline":{"value":0.5,"label":"Uniform random choice"},"lower":false,"range":null,"cases":800,"group":"retrieval","in_suite":false,"interactive":false},"55":{"id":55,"dataset":"Email spam classification","short":"Email spam","metric":"accuracy","explainer":"Detect spam in real emails, scored on the structured-focus question wording.","baseline":{"value":0.5,"label":"Uniform random choice"},"lower":false,"range":null,"cases":3000,"group":"retrieval","in_suite":false,"interactive":false},"56":{"id":56,"dataset":"PhishNChips phishing decisions","short":"PhishNChips","metric":"accuracy","explainer":"Says whether an email and its link are phishing or legitimate.","baseline":{"value":0.5,"label":"uniform random choice, mean of 1/options"},"lower":false,"range":null,"cases":2000,"group":"retrieval","in_suite":true,"interactive":false,"edition_added":"v0.2","references":[{"kind":"dataset","url":"https://huggingface.co/datasets/AreLit/PhishNChips"},{"kind":"code","url":"https://github.com/anisselbd/jev-phishing-bench"}]},"7":{"id":7,"dataset":"MiniWoB++","short":"MiniWoB++","metric":"episode success","explainer":"Interactive environment. No reproduction has run it, so it sits outside this edition of the index and appears only here.","baseline":{"value":null,"label":null},"lower":false,"cases":null,"group":"tools","in_suite":false,"interactive":true},"16":{"id":16,"dataset":"ScienceWorld","short":"ScienceWorld","metric":"episode success","explainer":"Interactive environment. No reproduction has run it, so it sits outside this edition of the index and appears only here.","baseline":{"value":null,"label":null},"lower":false,"cases":null,"group":"tools","in_suite":false,"interactive":true},"13":{"id":13,"dataset":"Boxoban","short":"Boxoban","metric":"episode success","explainer":"Interactive environment. No reproduction has run it, so it sits outside this edition of the index and appears only here.","baseline":{"value":null,"label":null},"lower":false,"cases":null,"group":"games","in_suite":false,"interactive":true},"15":{"id":15,"dataset":"RTFM","short":"RTFM","metric":"episode success","explainer":"Interactive environment. No reproduction has run it, so it sits outside this edition of the index and appears only here.","baseline":{"value":null,"label":null},"lower":false,"cases":null,"group":"games","in_suite":false,"interactive":true},"17":{"id":17,"dataset":"Hanabi","short":"Hanabi","metric":"episode success","explainer":"Interactive environment. No reproduction has run it, so it sits outside this edition of the index and appears only here.","baseline":{"value":null,"label":null},"lower":false,"cases":null,"group":"games","in_suite":false,"interactive":true},"19":{"id":19,"dataset":"Codenames","short":"Codenames","metric":"episode success","explainer":"Interactive environment. No reproduction has run it, so it sits outside this edition of the index and appears only here.","baseline":{"value":null,"label":null},"lower":false,"cases":null,"group":"games","in_suite":false,"interactive":true},"57":{"id":57,"dataset":"MMLU-Pro","short":"MMLU-Pro","metric":"accuracy","explainer":"Harder MMLU successor: questions across 14 subjects with up to ten options.","baseline":{"value":0.1109,"label":"uniform random choice, mean of 1/options"},"lower":false,"range":null,"cases":12032,"group":"knowledge","in_suite":true,"interactive":false,"edition_added":"v0.2","references":[{"kind":"paper","url":"https://arxiv.org/abs/2406.01574"},{"kind":"dataset","url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro"}],"gold":1.2},"58":{"id":58,"dataset":"BBH","short":"BBH","metric":"accuracy","explainer":"23 tricky reasoning tasks from BIG-Bench (logic, dates, tracking objects, fallacies), answered with no working.","baseline":{"value":0.3101,"label":"uniform random choice, mean of 1/options"},"lower":false,"range":null,"cases":5507,"group":"knowledge","in_suite":true,"interactive":false,"edition_added":"v0.2","references":[{"kind":"paper","url":"https://arxiv.org/abs/2210.09261"},{"kind":"dataset","url":"https://github.com/suzgunmirac/BIG-Bench-Hard"}],"gold":1.2},"62":{"id":62,"dataset":"When2Call","short":"When2Call","metric":"accuracy","explainer":"Given a question and an assistant's tools, picks the right move: answer, call a tool, ask for details, or decline.","baseline":{"value":0.25,"label":"uniform random choice, mean of 1/options"},"lower":false,"range":null,"cases":3652,"group":"tools","in_suite":true,"interactive":false,"edition_added":"v0.2","references":[{"kind":"paper","url":"https://aclanthology.org/2025.naacl-long.174"},{"kind":"dataset","url":"https://huggingface.co/datasets/nvidia/When2Call"}]},"64":{"id":64,"dataset":"New Yorker","short":"New Yorker","metric":"accuracy","explainer":"Given a text description of a New Yorker cartoon, picks the caption written for it from five.","baseline":{"value":0.2,"label":"uniform random choice, mean of 1/options"},"lower":false,"range":null,"cases":528,"group":"arts","in_suite":true,"interactive":false,"edition_added":"v0.2","references":[{"kind":"paper","url":"https://arxiv.org/abs/2209.06293"},{"kind":"dataset","url":"https://huggingface.co/datasets/jmhessel/newyorker_caption_contest"}]},"59":{"id":59,"dataset":"RAGTruth","short":"RAGTruth","metric":"F1 on hallucinated class","explainer":"Says yes or no: does an AI-written answer contain anything its source text doesn't support? Scored with F1.","baseline":{"value":0.5177,"label":"Always \"hallucinated\" · F1"},"lower":false,"range":null,"cases":2700,"group":"language","in_suite":true,"interactive":false,"edition_added":"v0.2","references":[{"kind":"dataset","url":"https://github.com/ParticleMedia/RAGTruth"}]},"61":{"id":61,"dataset":"HoVer","short":"HoVer","metric":"accuracy","explainer":"Decides whether a claim linking two to four Wikipedia articles is supported by their introductions.","baseline":{"value":0.5,"label":"uniform random choice, mean of 1/options"},"lower":false,"range":null,"cases":4000,"group":"retrieval","in_suite":true,"interactive":false,"edition_added":"v0.2","references":[{"kind":"paper","url":"https://arxiv.org/abs/2011.03088"},{"kind":"dataset","url":"https://github.com/hover-nlp/hover"}]}},"jev":{"engine":"jev","name":"Jev","version":"jev-1.13.0","edition":"Full release v1","results":{"7":{"score":1.0,"metric":"episode success","cases":9,"median_ms":null,"p95_ms":null,"usd":null},"16":{"score":0.067,"metric":"episode success","cases":10,"median_ms":null,"p95_ms":null,"usd":null},"13":{"score":0.0,"metric":"levels solved","cases":100,"median_ms":null,"p95_ms":null,"usd":null},"15":{"score":0.17,"metric":"episode success","cases":100,"median_ms":null,"p95_ms":null,"usd":null},"17":{"score":0.0,"metric":"episode success","cases":100,"median_ms":null,"p95_ms":null,"usd":null},"19":{"score":0.0,"metric":"episode success","cases":66,"median_ms":null,"p95_ms":null,"usd":null},"1":{"score":0.9575,"metric":"case exact accuracy","cases":1694,"median_ms":265.7,"p95_ms":382.1,"usd":0.1},"2":{"score":0.6528,"metric":"nDCG@10","cases":685,"median_ms":319.2,"p95_ms":494.7,"usd":0.274},"3":{"score":0.8819,"metric":"accuracy","cases":508,"median_ms":351.4,"p95_ms":461.7,"usd":0.216},"4":{"score":0.7974,"metric":"macro-F1","cases":3080,"median_ms":265.6,"p95_ms":439.1,"usd":0.222},"5":{"score":0.8927,"metric":"macro-F1","cases":5500,"median_ms":247.5,"p95_ms":399.3,"usd":0.644},"6":{"score":0.7992,"metric":"selected quality (quality objective)","cases":5001,"median_ms":238.9,"p95_ms":370.9,"usd":1.3},"9":{"score":0.5227,"metric":"case exact accuracy","cases":88,"median_ms":262.8,"p95_ms":356.1,"usd":0.012},"10":{"score":0.4295,"metric":"macro-F1","cases":2500,"median_ms":265.2,"p95_ms":457.0,"usd":0.19},"11":{"score":0.7169,"metric":"macro-F1","cases":123,"median_ms":306.2,"p95_ms":415.5,"usd":0.023},"12":{"score":0.7479,"metric":"macro-F1","cases":3200,"median_ms":259.2,"p95_ms":434.6,"usd":0.058},"20":{"score":0.906,"metric":"accuracy","cases":811,"median_ms":235.3,"p95_ms":392.4,"usd":0.091},"21":{"score":0.6187,"metric":"accuracy","cases":2628,"median_ms":261.9,"p95_ms":433.0,"usd":0.038},"22":{"score":0.181,"metric":"accuracy","cases":2000,"median_ms":301.8,"p95_ms":476.3,"usd":0.641},"23":{"score":0.6474,"metric":"accuracy","cases":5000,"median_ms":239.9,"p95_ms":400.8,"usd":0.192},"24":{"score":0.9173,"metric":"accuracy","cases":14042,"median_ms":232.5,"p95_ms":444.6,"usd":0.246},"25":{"score":0.7828,"metric":"accuracy","cases":198,"median_ms":253.8,"p95_ms":344.9,"usd":0.004},"26":{"score":0.9933,"metric":"accuracy","cases":2376,"median_ms":260.8,"p95_ms":411.8,"usd":0.036},"27":{"score":0.9778,"metric":"accuracy","cases":1172,"median_ms":258.1,"p95_ms":372.6,"usd":0.018},"28":{"score":0.9195,"metric":"accuracy","cases":1267,"median_ms":259.3,"p95_ms":374.6,"usd":0.018},"29":{"score":0.9452,"metric":"accuracy","cases":10042,"median_ms":231.0,"p95_ms":364.6,"usd":0.213},"30":{"score":0.7987,"metric":"accuracy","cases":1319,"median_ms":264.2,"p95_ms":412.1,"usd":0.055},"31":{"score":0.1722,"metric":"accuracy","cases":5000,"median_ms":241.0,"p95_ms":396.1,"usd":0.302},"32":{"score":0.6609,"metric":"accuracy","cases":752,"median_ms":255.7,"p95_ms":366.7,"usd":0.04},"33":{"score":0.2642,"metric":"case exact accuracy","cases":1650,"median_ms":267.3,"p95_ms":381.3,"usd":0.11},"36":{"score":0.4752,"metric":"nDCG@10","cases":220,"median_ms":405.7,"p95_ms":637.5,"usd":0.137},"37":{"score":0.5521,"metric":"macro-F1","cases":5000,"median_ms":236.5,"p95_ms":396.7,"usd":0.162},"38":{"score":0.2952,"metric":"per-review F1","cases":400,"median_ms":343.1,"p95_ms":523.9,"usd":0.474},"39":{"score":0.8698,"metric":"macro-F1","cases":979,"median_ms":259.2,"p95_ms":371.1,"usd":0.023},"40":{"score":0.5051,"metric":"Sarcasm F1 · track A, English","cases":1400,"median_ms":243.7,"p95_ms":401.3,"usd":0.08,"tracks":[{"track":"A · Arabic","score":0.4449,"headline":false},{"track":"A · English","score":0.5051,"headline":true},{"track":"C · Arabic pairs","score":0.7,"headline":false},{"track":"C · English pairs","score":0.95,"headline":false}]},"41":{"score":0.6463,"metric":"macro-F1","cases":3006,"median_ms":262.6,"p95_ms":437.5,"usd":0.057},"42":{"score":0.8406,"metric":"macro-F1","cases":5500,"median_ms":238.5,"p95_ms":399.4,"usd":0.221},"43":{"score":0.7298,"metric":"accuracy","cases":570,"median_ms":250.9,"p95_ms":360.9,"usd":0.011},"44":{"score":0.7264,"metric":"accuracy","cases":5000,"median_ms":238.0,"p95_ms":392.3,"usd":0.09},"45":{"score":0.2036,"metric":"accuracy","cases":501,"median_ms":253.7,"p95_ms":346.4,"usd":0.015},"48":{"score":0.1736,"metric":"Brier (lower is better)","cases":10139,"median_ms":233.9,"p95_ms":368.3,"usd":0.499},"50":{"score":0.4594,"metric":"accuracy","cases":1676,"median_ms":264.0,"p95_ms":377.9,"usd":0.084},"51":{"score":0.94,"metric":"accuracy","cases":500,"median_ms":1411.9,"p95_ms":2198.7,"usd":0.0},"52":{"score":0.8747,"metric":"accuracy","cases":1221,"median_ms":1387.5,"p95_ms":2570.8,"usd":0.0},"53":{"score":0.905,"metric":"accuracy","cases":300,"median_ms":1383.9,"p95_ms":2379.8,"usd":0.0},"54":{"score":0.9463,"metric":"accuracy","cases":800,"median_ms":1840.8,"p95_ms":3083.4,"usd":0.0},"55":{"score":0.9743,"metric":"accuracy","cases":3000,"median_ms":1382.0,"p95_ms":2622.4,"usd":0.0},"56":{"score":0.6255,"metric":"accuracy","cases":2000,"median_ms":1372.4,"p95_ms":null,"usd":0.077,"answered":2000,"edition_added":"v0.2"},"57":{"score":0.827,"metric":"accuracy","cases":12032,"median_ms":1053.5,"p95_ms":null,"usd":0.28,"answered":12032,"edition_added":"v0.2"},"58":{"score":0.9292,"metric":"accuracy","cases":5507,"median_ms":1032.4,"p95_ms":null,"usd":0.108,"answered":5507,"edition_added":"v0.2"},"62":{"score":0.8097,"metric":"accuracy","cases":3652,"median_ms":1047.2,"p95_ms":null,"usd":0.208,"answered":3652,"edition_added":"v0.2"},"64":{"score":0.7008,"metric":"accuracy","cases":528,"median_ms":1043.8,"p95_ms":null,"usd":0.011,"answered":528,"edition_added":"v0.2"},"59":{"score":0.7653,"metric":"F1 on hallucinated class","cases":2700,"median_ms":1039.7,"p95_ms":null,"usd":0.13,"answered":2700,"edition_added":"v0.2"},"61":{"score":0.7285,"metric":"accuracy","cases":4000,"median_ms":945.0,"p95_ms":null,"usd":0.127,"answered":4000,"edition_added":"v0.2"}},"latency":{"median":524.1,"p95":536.0,"note":"Hosted API, network round-trip from our lab: one sequential HTTPS client on the latency-v1 sample. Not comparable to the on-card single-process figures. Its answers on this sample differ from the Sep-19 board rows in 20 choices: API output drifted since the board run; scores are unchanged."},"interactive":[{"track":"miniwob","success_rate":1.0,"episodes":9},{"track":"minigrid_babyai","success_rate":0.0833,"episodes":12},{"track":"scienceworld","success_rate":0.0,"episodes":10},{"track":"hanabi","success_rate":null,"episodes":100},{"track":"leduc","success_rate":null,"episodes":300},{"track":"codenames","success_rate":0.0,"episodes":66},{"track":"rtfm","success_rate":0.17,"episodes":100}],"references":[{"catalog_id":25,"model":"Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback)","score":0.9374,"metric":"Accuracy"},{"catalog_id":45,"model":"Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback)","score":0.5913,"metric":"Accuracy"},{"catalog_id":25,"model":"GPT-6 Astra (max)","score":0.9606,"metric":"Accuracy"},{"catalog_id":45,"model":"GPT-6 Astra (max)","score":0.5468,"metric":"Accuracy"},{"catalog_id":25,"model":"Claude Opus 5 (Adaptive Reasoning, Max Effort)","score":0.9323,"metric":"Accuracy"},{"catalog_id":45,"model":"Claude Opus 5 (Adaptive Reasoning, Max Effort)","score":0.5487,"metric":"Accuracy"},{"catalog_id":25,"model":"Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)","score":0.9263,"metric":"Accuracy"},{"catalog_id":45,"model":"Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)","score":0.5547,"metric":"Accuracy"},{"catalog_id":25,"model":"Muse Spark 1.3 (max)","score":0.9354,"metric":"Accuracy"},{"catalog_id":45,"model":"Muse Spark 1.3 (max)","score":0.487,"metric":"Accuracy"},{"catalog_id":25,"model":"GPT-5.6 Sol (max)","score":0.9414,"metric":"Accuracy"},{"catalog_id":45,"model":"GPT-5.6 Sol (max)","score":0.4949,"metric":"Accuracy"},{"catalog_id":25,"model":"Qwen3.8 Max (0902)","score":0.9283,"metric":"Accuracy"},{"catalog_id":45,"model":"Qwen3.8 Max (0902)","score":0.431,"metric":"Accuracy"},{"catalog_id":25,"model":"GLM-5.3 (max)","score":0.9172,"metric":"Accuracy"},{"catalog_id":45,"model":"GLM-5.3 (max)","score":0.4226,"metric":"Accuracy"},{"catalog_id":25,"model":"Grok 4.6 (high)","score":0.9495,"metric":"Accuracy"},{"catalog_id":45,"model":"Grok 4.6 (high)","score":0.4291,"metric":"Accuracy"},{"catalog_id":25,"model":"Kimi K3 (max)","score":0.9354,"metric":"Accuracy"},{"catalog_id":45,"model":"Kimi K3 (max)","score":0.469,"metric":"Accuracy"},{"catalog_id":45,"model":"Step 5 Preview","score":0.4648,"metric":"Accuracy"},{"catalog_id":25,"model":"GPT-5.6 Terra (max)","score":0.9253,"metric":"Accuracy"},{"catalog_id":45,"model":"GPT-5.6 Terra (max)","score":0.4291,"metric":"Accuracy"},{"catalog_id":25,"model":"GLM 5.3 Flash","score":0.9121,"metric":"Accuracy"},{"catalog_id":45,"model":"GLM 5.3 Flash","score":0.3985,"metric":"Accuracy"},{"catalog_id":25,"model":"Gemini 3.8 Flash (high)","score":0.9525,"metric":"Accuracy"},{"catalog_id":45,"model":"Gemini 3.8 Flash (high)","score":0.4782,"metric":"Accuracy"},{"catalog_id":45,"model":"DeepSeek V4.1 Flash (Reasoning, Max Effort)","score":0.3925,"metric":"Accuracy"},{"catalog_id":34,"model":"Claude Fable 5","score":0.819,"metric":"AVG@5 accuracy"},{"catalog_id":34,"model":"Claude Opus 5","score":0.806,"metric":"AVG@5 accuracy"}],"summary_notes":["Metrics exclude failed/incomplete cases; coverage is explicit. No silent replacement of failures.","Different primary metrics are not averaged across benchmarks.","Latency includes HTTPS and concurrent service scheduling; not isolated model inference.","Full raw HTTP responses/payloads and all retry timings are in http-attempts.jsonl.","BPoMP/RouterBench/GSM8K and chunked tasks retain all linked requests for sampled cases."],"calibration":{"acc":0.7393,"conf":0.8115,"ece":0.074,"brier":0.3558,"over95":0.0207,"n":72594,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0002,"conf":0.082,"acc":0.4},{"lo":0.1,"w":0.0187,"conf":0.1649,"acc":0.0998},{"lo":0.2,"w":0.0312,"conf":0.2424,"acc":0.2013},{"lo":0.3,"w":0.0275,"conf":0.3493,"acc":0.3792},{"lo":0.4,"w":0.0317,"conf":0.4512,"acc":0.3783},{"lo":0.5,"w":0.0824,"conf":0.5464,"acc":0.4796},{"lo":0.6,"w":0.0808,"conf":0.6436,"acc":0.5217},{"lo":0.7,"w":0.085,"conf":0.7465,"acc":0.5904},{"lo":0.8,"w":0.1037,"conf":0.8488,"acc":0.708},{"lo":0.9,"w":0.5388,"conf":0.9808,"acc":0.9342}],"areas":{"knowledge":{"ece":0.0654,"acc":0.7156,"conf":0.7771},"language":{"ece":0.0617,"acc":0.8248,"conf":0.8862},"retrieval":{"ece":0.1536,"acc":0.6685,"conf":0.8206},"tools":{"ece":0.0196,"acc":0.9458,"conf":0.9534},"arts":{"ece":0.0934,"acc":0.5701,"conf":0.6605}}},"scores":{"balanced_raw":68.09,"balanced_skill":57.91,"breadth_skill":57.08},"frozen_scores":{"balanced_raw":54.47,"balanced_skill":43.61,"breadth_skill":36.98},"categories":[{"id":"knowledge","raw":0.6154,"skill":0.514,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.7334,"skill":0.6202,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.6825,"skill":0.5542,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.7764,"skill":0.7509,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.5368,"skill":0.3766,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9575,"skill":0.9426,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6528,"skill":0.5991,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7992,"skill":0.5266,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.786,"headline":false},{"track":"RouterBench-5shot","score":0.8125,"headline":false}]},"11":{"raw":0.7169,"skill":0.5906,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.9092,"skill":0.8184,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6187,"skill":0.2374,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.1658,"skill":0.1593,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.6441,"skill":0.2883,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.9199,"skill":0.8932,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.7857,"skill":0.7143,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.7987,"skill":0.7565,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.8218,"headline":false},{"track":"GSM8K-10choice","score":0.7756,"headline":false}]},"31":{"raw":0.1722,"skill":0.0984,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4752,"skill":0.4063,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.5521,"skill":0.4382,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.5051,"skill":0.3633,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4449,"headline":false},{"track":"A · English","score":0.5051,"headline":true},{"track":"C · Arabic pairs","score":0.7,"headline":false},{"track":"C · English pairs","score":0.95,"headline":false}]},"41":{"raw":0.6463,"skill":0.4695,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.7298,"skill":0.5713,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.7264,"skill":0.4528,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4594,"skill":0.2154,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":1.0,"skill":1.0,"coverage":1.0,"random":null,"tracks":[]},"16":{"raw":0.067,"skill":0.067,"coverage":1.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":null,"tracks":[]},"15":{"raw":0.17,"skill":0.17,"coverage":1.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":null,"tracks":[]},"56":{"raw":0.6255,"skill":0.251,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.827,"skill":0.8055,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.9292,"skill":0.8973,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.8097,"skill":0.7463,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.7008,"skill":0.6259,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.7653,"skill":0.5135,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.7285,"skill":0.457,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.6609,"skill":0.4609,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2642,"skill":0.2544,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.2036,"skill":0.0473,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.7479,"skill":0.6224,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.9195,"skill":0.839,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.9452,"skill":0.9269,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.2952,"skill":0.2727,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8698,"skill":0.8085,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.8406,"skill":0.6899,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.7974,"skill":0.7948,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.8927,"skill":0.8921,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.8819,"skill":0.8796,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.5227,"skill":0.5227,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.3056,"skill":0.3056,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":1.0,"pending":0.0},"models":[{"engine":"akash-gemma","name":"system-one-gemma","long_name":"system-one-gemma","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":268098176,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/akash-kamat/system-one-gemma","base_model":"google/gemma-3-270m","base_url":"https://huggingface.co/google/gemma-3-270m","weights_repo":null,"served_checkpoint":null},"completed":120340,"counts":{"ok":120237},"lifted":true,"calibration":{"acc":0.3215,"conf":0.5602,"ece":0.2387,"brier":0.7993,"over95":0.0161,"n":72594,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0429,"conf":0.0656,"acc":0.0395},{"lo":0.1,"w":0.0155,"conf":0.1554,"acc":0.0865},{"lo":0.2,"w":0.0652,"conf":0.2645,"acc":0.0665},{"lo":0.3,"w":0.1053,"conf":0.3525,"acc":0.2187},{"lo":0.4,"w":0.1291,"conf":0.4511,"acc":0.2903},{"lo":0.5,"w":0.2205,"conf":0.5478,"acc":0.3956},{"lo":0.6,"w":0.1573,"conf":0.6462,"acc":0.3549},{"lo":0.7,"w":0.1126,"conf":0.7462,"acc":0.3527},{"lo":0.8,"w":0.0829,"conf":0.8484,"acc":0.4576},{"lo":0.9,"w":0.0688,"conf":0.9534,"acc":0.479}],"areas":{"knowledge":{"ece":0.2116,"acc":0.2962,"conf":0.507},"language":{"ece":0.325,"acc":0.3207,"conf":0.6457},"retrieval":{"ece":0.2167,"acc":0.405,"conf":0.62},"tools":{"ece":0.2613,"acc":0.2167,"conf":0.478},"arts":{"ece":0.15,"acc":0.3745,"conf":0.5245}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":31.9,"p95":748.0,"mean":168.7},"results":{"1":{"score":0.3483,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":64.9},"2":{"score":0.1639,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1004.2},"3":{"score":0.0118,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.5347,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":47.6},"5":{"score":0.2486,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":86.5},"6":{"score":0.3575,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":549.7},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4519,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":44.5},"11":{"score":0.2072,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.2389,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":31.2},"20":{"score":0.5776,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":30.5},"21":{"score":0.4726,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":28.1},"22":{"score":0.0025,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":2918.6},"23":{"score":0.524,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":31.0},"24":{"score":0.3081,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":30.8},"25":{"score":0.2449,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.4465,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":30.4},"27":{"score":0.3097,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":30.4},"28":{"score":0.4988,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":28.1},"29":{"score":0.2841,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":31.5},"30":{"score":0.1892,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":32.0},"31":{"score":0.0912,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":60.1},"32":{"score":0.4162,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0012,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":233.8},"36":{"score":0.1033,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.1949,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":32.5},"38":{"score":0.0309,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1816.4},"39":{"score":0.13,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2506,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":28.1,"tracks":[{"track":"A · Arabic","score":0.2466,"headline":false},{"track":"A · English","score":0.2506,"headline":true},{"track":"C · Arabic pairs","score":0.335,"headline":false},{"track":"C · English pairs","score":0.345,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3489,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":29.5},"42":{"score":0.4253,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":31.0},"43":{"score":0.3404,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.502,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":28.8},"45":{"score":0.1257,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.3722,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":33.4},"50":{"score":0.2959,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":35.3},"56":{"score":0.5995,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":262.6,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.1317,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":31.7,"edition_added":"v0.2","jev":0.827},"58":{"score":0.2853,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":30.7,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.1569,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":34.5,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.2917,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.1293,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":33.4,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5052,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":32.1,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":25.36,"balanced_skill":5.07,"breadth_skill":4.17},"frozen_scores":{"balanced_raw":22.21,"balanced_skill":1.96,"breadth_skill":1.89},"categories":[{"id":"knowledge","raw":0.2301,"skill":0.0129,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.2593,"skill":0.0108,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.3581,"skill":0.1722,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.1394,"skill":0.0331,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.299,"skill":0.0408,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.3483,"skill":0.1202,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.1639,"skill":0.0344,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.3576,"skill":0.0,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.2032,"headline":false},{"track":"RouterBench-5shot","score":0.512,"headline":false}]},"11":{"raw":0.2072,"skill":0.0,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.5654,"skill":0.1309,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.4726,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0027,"skill":0.0,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5242,"skill":0.0484,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3064,"skill":0.0752,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2449,"skill":0.0,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.1892,"skill":0.0254,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.2881,"headline":false},{"track":"GSM8K-10choice","score":0.0902,"headline":false}]},"31":{"raw":0.0912,"skill":0.0102,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.1033,"skill":0.0,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.1949,"skill":0.0,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.2506,"skill":0.036,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2466,"headline":false},{"track":"A · English","score":0.2506,"headline":true},{"track":"C · Arabic pairs","score":0.335,"headline":false},{"track":"C · English pairs","score":0.345,"headline":false}]},"41":{"raw":0.3489,"skill":0.0234,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3404,"skill":0.0,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.502,"skill":0.004,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.2959,"skill":0.0,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5995,"skill":0.199,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.1317,"skill":0.0234,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.2853,"skill":0.0,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.1569,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.2917,"skill":0.1146,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.1293,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5052,"skill":0.0104,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.4162,"skill":0.0719,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0012,"skill":0.0,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1257,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.2389,"skill":0.0,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.4988,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.2841,"skill":0.0455,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0309,"skill":0.0,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.13,"skill":0.0,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.4253,"skill":0.0,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.5347,"skill":0.5287,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.2486,"skill":0.2441,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.0118,"skill":0.0,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author infer.score, fp32 (the only precision infer.load_trained_model offers), one padded batch of all options per question, SDPA default; max_len 32768 lift as board; OOM retry sub-batches of 8"},{"engine":"autojev-27b","name":"AutoJev-27B","long_name":"AutoJev-27B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":52173413928,"model_url":"https://huggingface.co/denis-pplx/autojev-27b","code_url":"https://github.com/denis-pplx/autojev","base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":"denis-pplx/autojev-27b","served_checkpoint":null},"completed":120340,"counts":{"ok":120340},"lifted":false,"calibration":{"acc":0.7295,"conf":0.7375,"ece":0.0178,"brier":0.361,"over95":0.0077,"n":72594,"benchmarks":32,"rel":[{"lo":0.0,"w":0.008,"conf":0.0763,"acc":0.2394},{"lo":0.1,"w":0.0312,"conf":0.1521,"acc":0.219},{"lo":0.2,"w":0.0333,"conf":0.2529,"acc":0.2714},{"lo":0.3,"w":0.0422,"conf":0.351,"acc":0.3724},{"lo":0.4,"w":0.0538,"conf":0.4507,"acc":0.4205},{"lo":0.5,"w":0.1138,"conf":0.5491,"acc":0.5165},{"lo":0.6,"w":0.1021,"conf":0.6481,"acc":0.6122},{"lo":0.7,"w":0.1086,"conf":0.7501,"acc":0.7242},{"lo":0.8,"w":0.1218,"conf":0.8513,"acc":0.8502},{"lo":0.9,"w":0.3852,"conf":0.9626,"acc":0.9602}],"areas":{"knowledge":{"ece":0.0232,"acc":0.6588,"conf":0.6763},"language":{"ece":0.0269,"acc":0.8402,"conf":0.8179},"retrieval":{"ece":0.0879,"acc":0.6703,"conf":0.7553},"tools":{"ece":0.027,"acc":0.9384,"conf":0.9404},"arts":{"ece":0.0695,"acc":0.6079,"conf":0.5912}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":101.4,"p95":1052.5,"mean":403.2},"results":{"1":{"score":0.9764,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":255.4},"2":{"score":0.6742,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":3413.7},"3":{"score":0.8406,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7903,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":172.9},"5":{"score":0.878,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":273.4},"6":{"score":0.7981,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":857.5},"9":{"score":0.7386,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.3976,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":217.7},"11":{"score":0.7815,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7065,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":98.2},"20":{"score":0.9366,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":98.5},"21":{"score":0.624,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":98.1},"22":{"score":0.388,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1011.6},"23":{"score":0.6546,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":117.0},"24":{"score":0.8401,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":98.7},"25":{"score":0.4949,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.992,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":97.6},"27":{"score":0.9676,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":97.9},"28":{"score":0.8516,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":99.1},"29":{"score":0.9398,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":99.1},"30":{"score":0.6736,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":99.0},"31":{"score":0.172,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":169.8},"32":{"score":0.6184,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2988,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":774.1},"36":{"score":0.4866,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5509,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":100.2},"38":{"score":0.204,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2022.7},"39":{"score":0.929,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.6034,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":98.3,"tracks":[{"track":"A · Arabic","score":0.4112,"headline":false},{"track":"A · English","score":0.6034,"headline":true},{"track":"C · Arabic pairs","score":0.78,"headline":false},{"track":"C · English pairs","score":0.965,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.7079,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":99.2},"42":{"score":0.8481,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":119.6},"43":{"score":0.7491,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.745,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":98.7},"45":{"score":0.1198,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.1948,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":151.9},"50":{"score":0.4177,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":170.4},"56":{"score":0.5995,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":515.1,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.6448,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":99.2,"edition_added":"v0.2","jev":0.827},"58":{"score":0.781,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":98.7,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.8168,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":145.1,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.7027,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.8034,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":146.2,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.7422,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":104.2,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":66.89,"balanced_skill":56.4,"breadth_skill":54.93},"frozen_scores":{"balanced_raw":49.4,"balanced_skill":38.56,"breadth_skill":32.16},"categories":[{"id":"knowledge","raw":0.5283,"skill":0.4088,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.7429,"skill":0.6346,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.6786,"skill":0.5486,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.8167,"skill":0.7935,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.5515,"skill":0.3939,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9764,"skill":0.9681,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6742,"skill":0.6237,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7981,"skill":0.5238,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7851,"headline":false},{"track":"RouterBench-5shot","score":0.8111,"headline":false}]},"11":{"raw":0.7815,"skill":0.684,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.9391,"skill":0.8782,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.624,"skill":0.2481,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.3781,"skill":0.3732,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.6442,"skill":0.2883,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8474,"skill":0.7965,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4949,"skill":0.3265,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.6736,"skill":0.6108,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.7612,"headline":false},{"track":"GSM8K-10choice","score":0.5861,"headline":false}]},"31":{"raw":0.172,"skill":0.0982,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4866,"skill":0.4192,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.5509,"skill":0.4366,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.6034,"skill":0.4898,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4112,"headline":false},{"track":"A · English","score":0.6034,"headline":true},{"track":"C · Arabic pairs","score":0.78,"headline":false},{"track":"C · English pairs","score":0.965,"headline":false}]},"41":{"raw":0.7079,"skill":0.5619,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.7491,"skill":0.602,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.745,"skill":0.49,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4177,"skill":0.1548,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5995,"skill":0.199,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.6448,"skill":0.6005,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.781,"skill":0.6826,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.8168,"skill":0.7557,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.7027,"skill":0.6284,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.8034,"skill":0.5924,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.7422,"skill":0.4844,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.6184,"skill":0.3933,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2988,"skill":0.2895,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1198,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.7065,"skill":0.5604,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.8516,"skill":0.7032,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.9398,"skill":0.9197,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.204,"skill":0.1785,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.929,"skill":0.8956,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.8481,"skill":0.7045,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.7903,"skill":0.7876,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.878,"skill":0.8773,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.8406,"skill":0.8375,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.7386,"skill":0.7386,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.2208,"skill":0.2208,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author DecisionModel forward (bf16, SDPA, one pass per question over the shared state) on one card; latency definition: In-process wall time of the DecisionModel forward: one pass per question over the shared state, CUDA-synchronized.","new_in_edition":true},{"engine":"bosun-v3.1-0.6b","name":"Bosun v3.1 0.6B","long_name":"Bosun v3.1 0.6B LoRA","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":596049920,"params_estimated":false,"params_from_name":false,"trained_bytes":41470928,"model_url":"https://huggingface.co/Hanno-Labs/bosun-v3.1-0.6b","code_url":"https://github.com/Hanno-Labs/jev-compatible-server","base_model":"Qwen/Qwen3-0.6B-Base","base_url":"https://huggingface.co/Qwen/Qwen3-0.6B-Base","weights_repo":"Hanno-Labs/bosun-v3.1-0.6b","served_checkpoint":null},"completed":120340,"counts":{"ok":119898},"lifted":false,"calibration":{"acc":0.4496,"conf":0.5883,"ece":0.1419,"brier":0.6671,"over95":0.0082,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0561,"conf":0.0453,"acc":0.0512},{"lo":0.1,"w":0.0272,"conf":0.1506,"acc":0.1913},{"lo":0.2,"w":0.0276,"conf":0.2547,"acc":0.2614},{"lo":0.3,"w":0.0891,"conf":0.3548,"acc":0.2501},{"lo":0.4,"w":0.1046,"conf":0.4491,"acc":0.3119},{"lo":0.5,"w":0.1894,"conf":0.5505,"acc":0.4418},{"lo":0.6,"w":0.1705,"conf":0.6471,"acc":0.5073},{"lo":0.7,"w":0.132,"conf":0.7465,"acc":0.5471},{"lo":0.8,"w":0.1027,"conf":0.8495,"acc":0.598},{"lo":0.9,"w":0.1006,"conf":0.9495,"acc":0.7503}],"areas":{"knowledge":{"ece":0.1633,"acc":0.3386,"conf":0.5018},"language":{"ece":0.1595,"acc":0.5205,"conf":0.68},"retrieval":{"ece":0.101,"acc":0.6176,"conf":0.6825},"tools":{"ece":0.21,"acc":0.5258,"conf":0.6823},"arts":{"ece":0.0962,"acc":0.386,"conf":0.4816}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":38.7,"p95":107.3,"mean":53.6},"results":{"1":{"score":0.4658,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":43.9},"2":{"score":0.5737,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":690.8},"3":{"score":0.5315,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7613,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":39.8},"5":{"score":0.6483,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":41.8},"6":{"score":0.6409,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":86.3},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5432,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":39.5},"11":{"score":0.2165,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.2749,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":38.6},"20":{"score":0.489,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":38.5},"21":{"score":0.5091,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":37.8},"22":{"score":0.035,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":95.0},"23":{"score":0.5094,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":37.9},"24":{"score":0.3468,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":38.1},"25":{"score":0.2755,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.5614,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":38.1},"27":{"score":0.4334,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":37.9},"28":{"score":0.4972,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":37.6},"29":{"score":0.3277,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":38.2},"30":{"score":0.1713,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":38.3},"31":{"score":0.0834,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39.3},"32":{"score":0.4428,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0145,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":76.4},"36":{"score":0.3171,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4299,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":38.7},"38":{"score":0.0556,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":249.0},"39":{"score":0.4799,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.0554,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":38.0,"tracks":[{"track":"A · Arabic","score":0.022,"headline":false},{"track":"A · English","score":0.0554,"headline":true},{"track":"C · Arabic pairs","score":0.455,"headline":false},{"track":"C · English pairs","score":0.47,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.2628,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":38.1},"42":{"score":0.4968,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":38.6},"43":{"score":0.3579,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.4948,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":38.4},"45":{"score":0.1497,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.289,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":39.2},"50":{"score":0.3663,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":39.5},"56":{"score":0.5,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":48.3,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.1774,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":38.3,"edition_added":"v0.2","jev":0.827},"58":{"score":0.3434,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":38.2,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.3001,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":39.6,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.3864,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.2917,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":39.7,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5055,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":38.7,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":35.07,"balanced_skill":14.32,"breadth_skill":11.02},"frozen_scores":{"balanced_raw":27.27,"balanced_skill":7.21,"breadth_skill":6.11},"categories":[{"id":"knowledge","raw":0.25,"skill":0.0283,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.2999,"skill":0.0383,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.5314,"skill":0.3415,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.3834,"skill":0.2844,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.3203,"skill":0.0557,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.4658,"skill":0.2788,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.5737,"skill":0.5077,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.6409,"skill":0.1556,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.6085,"headline":false},{"track":"RouterBench-5shot","score":0.6733,"headline":false}]},"11":{"raw":0.2165,"skill":0.0,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.4897,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5091,"skill":0.0183,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0333,"skill":0.0257,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5217,"skill":0.0434,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3579,"skill":0.1439,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2755,"skill":0.034,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.1713,"skill":0.0,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.2456,"headline":false},{"track":"GSM8K-10choice","score":0.097,"headline":false}]},"31":{"raw":0.0834,"skill":0.0017,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.3171,"skill":0.2274,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.4299,"skill":0.2849,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.0554,"skill":0.0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.022,"headline":false},{"track":"A · English","score":0.0554,"headline":true},{"track":"C · Arabic pairs","score":0.455,"headline":false},{"track":"C · English pairs","score":0.47,"headline":false}]},"41":{"raw":0.2628,"skill":0.0,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3579,"skill":0.0,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.4948,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3663,"skill":0.0804,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.1774,"skill":0.0748,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.3434,"skill":0.0483,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.3001,"skill":0.0668,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.3864,"skill":0.233,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.2917,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5055,"skill":0.011,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.4428,"skill":0.1141,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0145,"skill":0.0014,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1497,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.2749,"skill":0.0,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.4972,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.3277,"skill":0.1036,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0556,"skill":0.0254,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.4799,"skill":0.235,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.4968,"skill":0.021,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.7613,"skill":0.7582,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.6483,"skill":0.6462,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.5315,"skill":0.5225,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author RegistryRuntime.decide with the row-batched backend (all questions of a row scored together, prompts rendered exactly as model.predict); board used exact per-question mode; bf16, torch 2.14.0+cu130","new_in_edition":true},{"engine":"bosun-v3.1-1.7b","name":"Bosun v3.1 1.7B","long_name":"Bosun v3.1 1.7B LoRA","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":1720574976,"params_estimated":false,"params_from_name":false,"trained_bytes":71879720,"model_url":"https://huggingface.co/Hanno-Labs/bosun-v3.1-1.7b","code_url":"https://github.com/Hanno-Labs/jev-compatible-server","base_model":"Qwen/Qwen3-1.7B-Base","base_url":"https://huggingface.co/Qwen/Qwen3-1.7B-Base","weights_repo":"Hanno-Labs/bosun-v3.1-1.7b","served_checkpoint":null},"completed":120340,"counts":{"ok":119898},"lifted":false,"calibration":{"acc":0.5237,"conf":0.6534,"ece":0.1302,"brier":0.6136,"over95":0.0271,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.052,"conf":0.0413,"acc":0.0455},{"lo":0.1,"w":0.0217,"conf":0.1502,"acc":0.1487},{"lo":0.2,"w":0.024,"conf":0.2561,"acc":0.1817},{"lo":0.3,"w":0.0582,"conf":0.3536,"acc":0.2656},{"lo":0.4,"w":0.0887,"conf":0.4497,"acc":0.3267},{"lo":0.5,"w":0.1502,"conf":0.5499,"acc":0.4454},{"lo":0.6,"w":0.1267,"conf":0.6495,"acc":0.5456},{"lo":0.7,"w":0.1314,"conf":0.7507,"acc":0.6153},{"lo":0.8,"w":0.1399,"conf":0.8501,"acc":0.666},{"lo":0.9,"w":0.2071,"conf":0.9604,"acc":0.7687}],"areas":{"knowledge":{"ece":0.1726,"acc":0.3791,"conf":0.5517},"language":{"ece":0.1211,"acc":0.6198,"conf":0.7409},"retrieval":{"ece":0.1299,"acc":0.7441,"conf":0.8238},"tools":{"ece":0.0725,"acc":0.6822,"conf":0.7122},"arts":{"ece":0.1509,"acc":0.3973,"conf":0.5481}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":40.3,"p95":173.5,"mean":68.1},"results":{"1":{"score":0.6021,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":45.0},"2":{"score":0.5699,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1117.0},"3":{"score":0.6083,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8561,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":41.6},"5":{"score":0.9088,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":62.4},"6":{"score":0.6827,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":137.6},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5472,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":40.9},"11":{"score":0.4436,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.4048,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":39.8},"20":{"score":0.5522,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":40.1},"21":{"score":0.5384,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":39.7},"22":{"score":0.015,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":163.8},"23":{"score":0.5546,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39.9},"24":{"score":0.4235,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":39.5},"25":{"score":0.25,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.6654,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":39.4},"27":{"score":0.5358,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":39.6},"28":{"score":0.5178,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":39.6},"29":{"score":0.3488,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":39.8},"30":{"score":0.1494,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":40.0},"31":{"score":0.091,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":40.7},"32":{"score":0.4176,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0394,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":127.2},"36":{"score":0.3289,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4784,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":40.4},"38":{"score":0.0879,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":416.7},"39":{"score":0.7229,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.1538,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":39.9,"tracks":[{"track":"A · Arabic","score":0.0181,"headline":false},{"track":"A · English","score":0.1538,"headline":true},{"track":"C · Arabic pairs","score":0.45,"headline":false},{"track":"C · English pairs","score":0.445,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3642,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":40.0},"42":{"score":0.5324,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":40.5},"43":{"score":0.4018,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.532,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39.9},"45":{"score":0.1337,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.3305,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":40.6},"50":{"score":0.361,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":41.3},"56":{"score":0.502,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":76.2,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.2463,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":39.9,"edition_added":"v0.2","jev":0.827},"58":{"score":0.3684,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":39.7,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.3237,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":40.7,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.3598,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.2212,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":40.9,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5363,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":40.4,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":40.1,"balanced_skill":20.1,"breadth_skill":16.55},"frozen_scores":{"balanced_raw":30.41,"balanced_skill":11.06,"breadth_skill":9.44},"categories":[{"id":"knowledge","raw":0.262,"skill":0.0472,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.3822,"skill":0.124,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.6105,"skill":0.4282,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.4345,"skill":0.3478,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.3283,"skill":0.0747,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.6021,"skill":0.4629,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.5699,"skill":0.5033,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.6827,"skill":0.2594,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.6344,"headline":false},{"track":"RouterBench-5shot","score":0.7311,"headline":false}]},"11":{"raw":0.4436,"skill":0.1953,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.5533,"skill":0.1066,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5384,"skill":0.0769,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.013,"skill":0.0053,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5383,"skill":0.0766,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4371,"skill":0.2494,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.25,"skill":0.0,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.1494,"skill":0.0,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.2108,"headline":false},{"track":"GSM8K-10choice","score":0.0879,"headline":false}]},"31":{"raw":0.091,"skill":0.01,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.3289,"skill":0.2408,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.4784,"skill":0.3457,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.1538,"skill":0.0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0181,"headline":false},{"track":"A · English","score":0.1538,"headline":true},{"track":"C · Arabic pairs","score":0.45,"headline":false},{"track":"C · English pairs","score":0.445,"headline":false}]},"41":{"raw":0.3642,"skill":0.0463,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4018,"skill":0.0508,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.532,"skill":0.064,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.361,"skill":0.0726,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.502,"skill":0.004,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.2463,"skill":0.1523,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.3684,"skill":0.0845,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.3237,"skill":0.0983,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.3598,"skill":0.1997,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.2212,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5363,"skill":0.0726,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.4176,"skill":0.0741,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0394,"skill":0.0266,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1337,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.4048,"skill":0.1084,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5178,"skill":0.0356,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.3488,"skill":0.1317,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0879,"skill":0.0587,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.7229,"skill":0.5924,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.5324,"skill":0.0903,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.8561,"skill":0.8542,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.9088,"skill":0.9082,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.6083,"skill":0.6008,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author RegistryRuntime.decide with the row-batched backend (all questions of a row scored together, prompts rendered exactly as model.predict); board used exact per-question mode; bf16, torch 2.14.0+cu130","new_in_edition":true},{"engine":"clm-v0.1-8b","name":"CLM-v0.1-8B","long_name":"CLM-v0.1-8B Qwen3-8B heads","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":8190735360,"params_estimated":false,"params_from_name":false,"trained_bytes":75557149,"model_url":"https://huggingface.co/Contrastive-LM/CLM-v0.1-8B","code_url":"https://github.com/Contrastive-LM/CLM","base_model":"Qwen/Qwen3-8B-Base","base_url":"https://huggingface.co/Qwen/Qwen3-8B-Base","weights_repo":"Contrastive-LM/CLM-v0.1-8B","served_checkpoint":"Contrastive-LM/CLM-v0.1-8B"},"completed":120340,"counts":{"ok":119898},"lifted":false,"calibration":{"acc":0.3582,"conf":0.6809,"ece":0.3227,"brier":0.9042,"over95":0.1194,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0011,"conf":0.0925,"acc":0.0332},{"lo":0.1,"w":0.0361,"conf":0.1671,"acc":0.05},{"lo":0.2,"w":0.0512,"conf":0.242,"acc":0.0698},{"lo":0.3,"w":0.0707,"conf":0.361,"acc":0.1182},{"lo":0.4,"w":0.068,"conf":0.4559,"acc":0.2311},{"lo":0.5,"w":0.1531,"conf":0.5496,"acc":0.39},{"lo":0.6,"w":0.1268,"conf":0.6502,"acc":0.4558},{"lo":0.7,"w":0.1105,"conf":0.7488,"acc":0.4026},{"lo":0.8,"w":0.1121,"conf":0.8508,"acc":0.4863},{"lo":0.9,"w":0.2704,"conf":0.966,"acc":0.4149}],"areas":{"knowledge":{"ece":0.2685,"acc":0.3857,"conf":0.6542},"language":{"ece":0.3593,"acc":0.4277,"conf":0.7865},"retrieval":{"ece":0.4925,"acc":0.1219,"conf":0.6144},"tools":{"ece":0.234,"acc":0.3779,"conf":0.5752},"arts":{"ece":0.3165,"acc":0.3497,"conf":0.6662}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":46.8,"p95":130.9,"mean":56.5},"results":{"1":{"score":0.389,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":62.0},"2":{"score":0.4031,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":289.9},"3":{"score":0.061,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.0652,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":19.2},"5":{"score":0.0889,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":20.5},"6":{"score":0.3932,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":109.4},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.1001,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":68.1},"11":{"score":0.2011,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.3156,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":18.8},"20":{"score":0.3294,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":30.3},"21":{"score":0.5556,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":29.6},"22":{"score":0,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":120.1},"23":{"score":0.508,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":62.6},"24":{"score":0.358,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":46.5},"25":{"score":0.2041,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.564,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":45.4},"27":{"score":0.4147,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":45.6},"28":{"score":0.5083,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":44.5},"29":{"score":0.4035,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":50.3},"30":{"score":0.2316,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":48.3},"31":{"score":0.0818,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":61.5},"32":{"score":0.3457,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0558,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":67.3},"36":{"score":0.2915,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.0497,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.5},"38":{"score":0.031,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":195.9},"39":{"score":0.3335,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.3122,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":16.9,"tracks":[{"track":"A · Arabic","score":0.3401,"headline":false},{"track":"A · English","score":0.3122,"headline":true},{"track":"C · Arabic pairs","score":0.52,"headline":false},{"track":"C · English pairs","score":0.57,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.2271,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":20.5},"42":{"score":0.5649,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":35.0},"43":{"score":0.4281,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5394,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":18.7},"45":{"score":0.1577,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2815,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":42.5},"50":{"score":0.3711,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":72.1},"56":{"score":0.6505,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":49.4,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.1466,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":50.3,"edition_added":"v0.2","jev":0.827},"58":{"score":0.308,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":46.8,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.3869,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":67.5,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.2538,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.5177,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":41.0,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.52,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":29.6,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":27.94,"balanced_skill":7.4,"breadth_skill":6.94},"frozen_scores":{"balanced_raw":24.04,"balanced_skill":5.74,"breadth_skill":5.38},"categories":[{"id":"knowledge","raw":0.2465,"skill":0.0311,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.3453,"skill":0.0523,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.2659,"skill":0.1126,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.2463,"skill":0.1398,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.2812,"skill":0.043,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.389,"skill":0.1752,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.4031,"skill":0.3107,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.3932,"skill":0.0,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.2747,"headline":false},{"track":"RouterBench-5shot","score":0.5118,"headline":false}]},"11":{"raw":0.2011,"skill":0.0,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.3226,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5556,"skill":0.1111,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0,"skill":0.0,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5219,"skill":0.0437,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3646,"skill":0.1529,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2041,"skill":0.0,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.2316,"skill":0.0735,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.345,"headline":false},{"track":"GSM8K-10choice","score":0.1183,"headline":false}]},"31":{"raw":0.0818,"skill":0.0,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.2915,"skill":0.1985,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.0497,"skill":0.0,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.3122,"skill":0.1152,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3401,"headline":false},{"track":"A · English","score":0.3122,"headline":true},{"track":"C · Arabic pairs","score":0.52,"headline":false},{"track":"C · English pairs","score":0.57,"headline":false}]},"41":{"raw":0.2271,"skill":0.0,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4281,"skill":0.0926,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5394,"skill":0.0788,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3711,"skill":0.0873,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.6505,"skill":0.301,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.1466,"skill":0.0402,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.308,"skill":0.0,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.3869,"skill":0.1825,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.2538,"skill":0.0673,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.5177,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.52,"skill":0.04,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.3457,"skill":0.0,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0558,"skill":0.0433,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1577,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.3156,"skill":0.0,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5083,"skill":0.0166,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.4035,"skill":0.2047,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.031,"skill":0.0,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.3335,"skill":0.0197,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.5649,"skill":0.1535,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.0652,"skill":0.0532,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.0889,"skill":0.0834,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.061,"skill":0.0429,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"clm.Engine.answer against the owned vLLM pooling server with CUDA graphs (JEV_CLM_GRAPHS=1; the author script uses --enforce-eager: eager measured 58.5 / 133.2 ms vs graphs 46.8 / 130.9), prefix caching on, gpu util 0.85; eager run kept as latency-v1-full.eager","new_in_edition":true},{"engine":"decider-2b-fp8-http","name":"Decider 2B","long_name":"Decider 2B","variant":"FP8","meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":2274069824,"params_estimated":false,"params_from_name":false,"trained_bytes":3763692048,"model_url":"https://huggingface.co/Mapika/decider-2b","code_url":null,"base_model":"Qwen/Qwen3.5-2B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-2B-Base","weights_repo":"Mapika/decider-2b","served_checkpoint":"Mapika/decider-2b"},"completed":120340,"counts":{"ok":120340},"lifted":false,"calibration":{"acc":0.5859,"conf":0.663,"ece":0.0771,"brier":0.52,"over95":0.0097,"n":72594,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0516,"conf":0.0575,"acc":0.0288},{"lo":0.1,"w":0.0298,"conf":0.1499,"acc":0.1282},{"lo":0.2,"w":0.0273,"conf":0.2553,"acc":0.1883},{"lo":0.3,"w":0.059,"conf":0.3525,"acc":0.2861},{"lo":0.4,"w":0.0691,"conf":0.455,"acc":0.3442},{"lo":0.5,"w":0.1369,"conf":0.5491,"acc":0.4997},{"lo":0.6,"w":0.1238,"conf":0.6498,"acc":0.5915},{"lo":0.7,"w":0.1297,"conf":0.7494,"acc":0.6396},{"lo":0.8,"w":0.1325,"conf":0.8508,"acc":0.742},{"lo":0.9,"w":0.2403,"conf":0.9603,"acc":0.8816}],"areas":{"knowledge":{"ece":0.0802,"acc":0.5128,"conf":0.593},"language":{"ece":0.0597,"acc":0.7007,"conf":0.7604},"retrieval":{"ece":0.1054,"acc":0.6542,"conf":0.7537},"tools":{"ece":0.0671,"acc":0.6702,"conf":0.7337},"arts":{"ece":0.0917,"acc":0.4347,"conf":0.5263}}},"gaps":{"share":0.0,"reasons":[{"reason":"runtime error","share":0.0}]},"latency":{"median":8.1,"p95":134.9,"mean":40.5},"results":{"1":{"score":0.8294,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":21.6},"2":{"score":0.6312,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":150.3},"3":{"score":0.3169,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7277,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":13.1},"5":{"score":0.7369,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":36.3},"6":{"score":0.7988,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":132.3},"9":{"score":0.0341,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4885,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":40.4},"11":{"score":0.634,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.4937,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":6.3},"20":{"score":0.6492,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":6.4},"21":{"score":0.5993,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":5.8},"22":{"score":0.0025,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":154.8},"23":{"score":0.5696,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":10.8},"24":{"score":0.619,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":6.1},"25":{"score":0.3622,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9377,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":6.0},"27":{"score":0.8515,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":5.9},"28":{"score":0.6772,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":5.8},"29":{"score":0.8581,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":7.0},"30":{"score":0.2111,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":6.8},"31":{"score":0.0988,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.7},"32":{"score":0.5492,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2467,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":75.9},"36":{"score":0.3711,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3468,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":8.2},"38":{"score":0.0786,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":110.2},"39":{"score":0.7612,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2243,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":6.0,"tracks":[{"track":"A · Arabic","score":0.2174,"headline":false},{"track":"A · English","score":0.2243,"headline":true},{"track":"C · Arabic pairs","score":0.375,"headline":false},{"track":"C · English pairs","score":0.55,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.5104,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":6.7},"42":{"score":0.6547,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":14.7},"43":{"score":0.3298,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5726,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":6.3},"45":{"score":0.1198,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2706,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":13.8},"50":{"score":0.4218,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":15.0},"56":{"score":0.5005,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":38.2,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.3758,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":6.8,"edition_added":"v0.2","jev":0.827},"58":{"score":0.5097,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":7.0,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.5471,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":14.9,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.447,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.2835,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":14.4,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5907,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":10.4,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":45.85,"balanced_skill":28.97,"breadth_skill":27.29},"frozen_scores":{"balanced_raw":35.59,"balanced_skill":18.92,"breadth_skill":16.02},"categories":[{"id":"knowledge","raw":0.3379,"skill":0.1493,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.5266,"skill":0.3257,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.5516,"skill":0.3729,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.4793,"skill":0.4245,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.3699,"skill":0.1464,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.8294,"skill":0.7697,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6312,"skill":0.5741,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7988,"skill":0.5256,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7853,"headline":false},{"track":"RouterBench-5shot","score":0.8123,"headline":false}]},"11":{"raw":0.634,"skill":0.4706,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.6481,"skill":0.2962,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5993,"skill":0.1986,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.002,"skill":0.0,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5449,"skill":0.0897,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6439,"skill":0.5253,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3622,"skill":0.1497,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.2111,"skill":0.0446,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.2904,"headline":false},{"track":"GSM8K-10choice","score":0.1319,"headline":false}]},"31":{"raw":0.0988,"skill":0.0185,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.3711,"skill":0.2885,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.3468,"skill":0.1808,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.2243,"skill":0.0021,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2174,"headline":false},{"track":"A · English","score":0.2243,"headline":true},{"track":"C · Arabic pairs","score":0.375,"headline":false},{"track":"C · English pairs","score":0.55,"headline":false}]},"41":{"raw":0.5104,"skill":0.2656,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3298,"skill":0.0,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5726,"skill":0.1452,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4218,"skill":0.1609,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5005,"skill":0.001,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.3758,"skill":0.2979,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.5097,"skill":0.2893,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.5471,"skill":0.3961,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.447,"skill":0.3087,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.2835,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5907,"skill":0.1814,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5492,"skill":0.2833,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2467,"skill":0.2367,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1198,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.4937,"skill":0.2416,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.6772,"skill":0.3544,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.8581,"skill":0.8108,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0786,"skill":0.0491,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.7612,"skill":0.6488,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.6547,"skill":0.3282,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.7277,"skill":0.7242,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.7369,"skill":0.7353,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.3169,"skill":0.3037,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0341,"skill":0.0341,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author decider.serve through decider_memory_app (DECIDER_COMPILE=1, DECIDER_FP8=1, CUDA graphs, DECIDER_MAX_FWD_TOKENS 65536), same server as board; one sequential HTTP client (run_decider_http_suite.py --concurrency 1) instead of concurrency 8"},{"engine":"decider-35b-nvfp4","name":"Decider 35B-A3B","long_name":"Decider 35B-A3B","variant":"NVFP4","meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":35951822704,"params_estimated":false,"params_from_name":false,"trained_bytes":21013206328,"model_url":"https://huggingface.co/Mapika/decider-35b-a3b-nvfp4","code_url":null,"base_model":"Qwen/Qwen3.5-35B-A3B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B-Base","weights_repo":"Mapika/decider-35b-a3b-nvfp4","served_checkpoint":"Mapika/decider-35b-a3b-nvfp4"},"completed":120340,"counts":{"ok":120340},"lifted":false,"calibration":{"acc":0.6959,"conf":0.6757,"ece":0.0226,"brier":0.3878,"over95":0.0038,"n":72594,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0446,"conf":0.0708,"acc":0.0496},{"lo":0.1,"w":0.0314,"conf":0.1412,"acc":0.1385},{"lo":0.2,"w":0.0297,"conf":0.2567,"acc":0.3389},{"lo":0.3,"w":0.063,"conf":0.3522,"acc":0.3491},{"lo":0.4,"w":0.073,"conf":0.449,"acc":0.4597},{"lo":0.5,"w":0.1323,"conf":0.5494,"acc":0.5848},{"lo":0.6,"w":0.1206,"conf":0.6518,"acc":0.6765},{"lo":0.7,"w":0.1057,"conf":0.7497,"acc":0.7962},{"lo":0.8,"w":0.1065,"conf":0.8493,"acc":0.8918},{"lo":0.9,"w":0.2932,"conf":0.9704,"acc":0.9742}],"areas":{"knowledge":{"ece":0.0283,"acc":0.6093,"conf":0.6242},"language":{"ece":0.0387,"acc":0.8224,"conf":0.7843},"retrieval":{"ece":0.0538,"acc":0.7811,"conf":0.7372},"tools":{"ece":0.065,"acc":0.7947,"conf":0.7368},"arts":{"ece":0.0347,"acc":0.5312,"conf":0.5078}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":101.4,"p95":210.9,"mean":109.5},"results":{"1":{"score":0.9215,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":198.4},"2":{"score":0.663,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":370.2},"3":{"score":0.565,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8979,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":100.5},"5":{"score":0.9468,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":102.1},"6":{"score":0.7963,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":207.8},"9":{"score":0.2386,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6414,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":105.0},"11":{"score":0.7496,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6603,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":101.1},"20":{"score":0.8906,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":102.4},"21":{"score":0.6195,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":41.3},"22":{"score":0.0665,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":186.8},"23":{"score":0.649,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":99.8},"24":{"score":0.8261,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":40.8},"25":{"score":0.4082,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9899,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":41.2},"27":{"score":0.9633,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":40.2},"28":{"score":0.8863,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":40.7},"29":{"score":0.9665,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":100.0},"30":{"score":0.4606,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":102.2},"31":{"score":0.1088,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":101.0},"32":{"score":0.6184,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3473,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":217.8},"36":{"score":0.4167,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5065,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":102.6},"38":{"score":0.2724,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":319.2},"39":{"score":0.8524,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5481,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":41.0,"tracks":[{"track":"A · Arabic","score":0.2937,"headline":false},{"track":"A · English","score":0.5481,"headline":true},{"track":"C · Arabic pairs","score":0.535,"headline":false},{"track":"C · English pairs","score":0.845,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.5452,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":100.7},"42":{"score":0.7847,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":102.6},"43":{"score":0.5982,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6672,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":101.2},"45":{"score":0.0878,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2007,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":101.4},"50":{"score":0.4642,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":103.7},"56":{"score":0.6685,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":206.6,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.6046,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":100.8,"edition_added":"v0.2","jev":0.827},"58":{"score":0.7133,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":99.3,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.5972,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":101.7,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.6402,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.631,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":103.3,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.635,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":101.7,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":59.72,"balanced_skill":47.11,"breadth_skill":45.91},"frozen_scores":{"balanced_raw":43.6,"balanced_skill":30.59,"breadth_skill":25.7},"categories":[{"id":"knowledge","raw":0.4608,"skill":0.3178,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.698,"skill":0.5552,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.6854,"skill":0.5466,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.6079,"skill":0.5654,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4924,"skill":0.326,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9215,"skill":0.894,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.663,"skill":0.6108,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7963,"skill":0.5196,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7832,"headline":false},{"track":"RouterBench-5shot","score":0.8094,"headline":false}]},"11":{"raw":0.7496,"skill":0.6379,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.8918,"skill":0.7836,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6195,"skill":0.239,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0635,"skill":0.0562,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.6297,"skill":0.2594,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8331,"skill":0.7775,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4082,"skill":0.2109,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.4606,"skill":0.3531,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.5724,"headline":false},{"track":"GSM8K-10choice","score":0.3487,"headline":false}]},"31":{"raw":0.1088,"skill":0.0293,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4167,"skill":0.3401,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.5065,"skill":0.381,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.5481,"skill":0.4187,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2937,"headline":false},{"track":"A · English","score":0.5481,"headline":true},{"track":"C · Arabic pairs","score":0.535,"headline":false},{"track":"C · English pairs","score":0.845,"headline":false}]},"41":{"raw":0.5452,"skill":0.3178,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.5982,"skill":0.3626,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6672,"skill":0.3344,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4642,"skill":0.2224,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.6685,"skill":0.337,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.6046,"skill":0.5553,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.7133,"skill":0.5844,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.5972,"skill":0.4629,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.6402,"skill":0.5502,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.631,"skill":0.2349,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.635,"skill":0.27,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.6184,"skill":0.3933,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.3473,"skill":0.3386,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.0878,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.6603,"skill":0.4912,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.8863,"skill":0.7726,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.9665,"skill":0.9553,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.2724,"skill":0.2491,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8524,"skill":0.7829,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7847,"skill":0.5811,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.8979,"skill":0.8966,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.9468,"skill":0.9465,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.565,"skill":0.5566,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.2386,"skill":0.2386,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.1972,"skill":0.1972,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"in-process vLLM 0.29.0 LLM, NVFP4 MoE, CUDA graphs, prefix caching off (board config; a prefix-cache run was slower and flipped answers), gpu util 0.9, venv-vllm torch 2.13.0+cu130"},{"engine":"decider-4b","name":"Decider 4B","long_name":"Decider 4B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":8411558400,"model_url":"https://huggingface.co/Mapika/decider-4b","code_url":"https://github.com/Mapika/decider","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"Mapika/decider-4b","served_checkpoint":null},"completed":120340,"counts":{"ok":119898},"lifted":false,"calibration":{"acc":0.6488,"conf":0.7326,"ece":0.0837,"brier":0.4542,"over95":0.0178,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0381,"conf":0.0703,"acc":0.0666},{"lo":0.1,"w":0.0214,"conf":0.1363,"acc":0.1303},{"lo":0.2,"w":0.0163,"conf":0.2568,"acc":0.2542},{"lo":0.3,"w":0.0444,"conf":0.3544,"acc":0.2521},{"lo":0.4,"w":0.0582,"conf":0.4494,"acc":0.3416},{"lo":0.5,"w":0.1145,"conf":0.5487,"acc":0.471},{"lo":0.6,"w":0.1007,"conf":0.6494,"acc":0.5228},{"lo":0.7,"w":0.101,"conf":0.752,"acc":0.6267},{"lo":0.8,"w":0.12,"conf":0.8523,"acc":0.7324},{"lo":0.9,"w":0.3853,"conf":0.9716,"acc":0.9095}],"areas":{"knowledge":{"ece":0.0964,"acc":0.5764,"conf":0.6706},"language":{"ece":0.0434,"acc":0.7858,"conf":0.8292},"retrieval":{"ece":0.1502,"acc":0.6088,"conf":0.7468},"tools":{"ece":0.0634,"acc":0.8063,"conf":0.8675},"arts":{"ece":0.1048,"acc":0.4994,"conf":0.6026}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":12.6,"p95":188.5,"mean":41.0},"results":{"1":{"score":0.8796,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":33.7},"2":{"score":0.65,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":288.9},"3":{"score":0.6043,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7472,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":21.9},"5":{"score":0.8655,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":44.0},"6":{"score":0.7988,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":172.2},"9":{"score":0.1705,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.2003,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":45.2},"11":{"score":0.7234,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5636,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":9.2},"20":{"score":0.8344,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":9.2},"21":{"score":0.5967,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":8.1},"22":{"score":0.1,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":242.7},"23":{"score":0.6048,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":15.3},"24":{"score":0.7418,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":8.5},"25":{"score":0.4133,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9781,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":8.3},"27":{"score":0.9394,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":8.4},"28":{"score":0.8074,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":7.9},"29":{"score":0.9399,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":9.5},"30":{"score":0.5709,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":9.3},"31":{"score":0.105,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.7},"32":{"score":0.5372,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3158,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":127.2},"36":{"score":0.4545,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4238,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":12.6},"38":{"score":0.1924,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":234.8},"39":{"score":0.8604,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4207,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":8.2,"tracks":[{"track":"A · Arabic","score":0.0829,"headline":false},{"track":"A · English","score":0.4207,"headline":true},{"track":"C · Arabic pairs","score":0.595,"headline":false},{"track":"C · English pairs","score":0.875,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.5353,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":9.2},"42":{"score":0.7369,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":22.2},"43":{"score":0.4368,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.633,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":9.2},"45":{"score":0.1038,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.294,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":19.6},"50":{"score":0.4063,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":22.9},"56":{"score":0.578,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":79.0,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.5165,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":9.4,"edition_added":"v0.2","jev":0.827},"58":{"score":0.6089,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":9.2,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.7588,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":22.6,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.6288,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.5216,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":21.1,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5627,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":13.7,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":55.45,"balanced_skill":40.7,"breadth_skill":39.17},"frozen_scores":{"balanced_raw":41.59,"balanced_skill":27.46,"breadth_skill":23.2},"categories":[{"id":"knowledge","raw":0.4231,"skill":0.2574,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.6381,"skill":0.4597,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.6129,"skill":0.4467,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.6222,"skill":0.5862,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4374,"skill":0.2503,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.8796,"skill":0.8374,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.65,"skill":0.5959,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7989,"skill":0.5257,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7854,"headline":false},{"track":"RouterBench-5shot","score":0.8123,"headline":false}]},"11":{"raw":0.7234,"skill":0.6,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.833,"skill":0.666,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5967,"skill":0.1933,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0934,"skill":0.0863,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5913,"skill":0.1825,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7598,"skill":0.6798,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4133,"skill":0.2177,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.5709,"skill":0.4832,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.6399,"headline":false},{"track":"GSM8K-10choice","score":0.5019,"headline":false}]},"31":{"raw":0.105,"skill":0.0252,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4545,"skill":0.3829,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.4238,"skill":0.2773,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.4207,"skill":0.2548,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0829,"headline":false},{"track":"A · English","score":0.4207,"headline":true},{"track":"C · Arabic pairs","score":0.595,"headline":false},{"track":"C · English pairs","score":0.875,"headline":false}]},"41":{"raw":0.5353,"skill":0.3029,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4368,"skill":0.1065,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.633,"skill":0.266,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4063,"skill":0.1384,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.578,"skill":0.156,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.5165,"skill":0.4562,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6089,"skill":0.4331,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.7588,"skill":0.6784,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.6288,"skill":0.536,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.5216,"skill":0.0081,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5627,"skill":0.1254,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5372,"skill":0.2642,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.3158,"skill":0.3067,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1038,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.5636,"skill":0.3463,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.8074,"skill":0.6148,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.9399,"skill":0.9199,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.1924,"skill":0.1666,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8604,"skill":0.7947,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7369,"skill":0.4881,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.7472,"skill":0.7439,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.8655,"skill":0.8647,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.6043,"skill":0.5967,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.1705,"skill":0.1705,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author decider.serve v1.2.1 with DECIDER_FP8=1 and DECIDER_COMPILE=1 on top of CUDA graphs (89 captured), one sequential HTTP client; the board ran graphs only. FP8+compile: 12.6 / 188.5 ms; graphs only: 13.6 / 172.7 ms (kept as latency-v1-full.stock). Chosen on the median"},{"engine":"decider-chat-gemma4-31b","name":"Decider chat · Gemma-4-31B","long_name":"Decider chat · Gemma-4-31B","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":32682372656,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/Mapika/decider","base_model":"google/gemma-4-31B","base_url":"https://huggingface.co/google/gemma-4-31B","weights_repo":null,"served_checkpoint":"google/gemma-4-31B-it"},"completed":120340,"counts":{"ok":119888},"lifted":false,"calibration":{"acc":0.724,"conf":0.7711,"ece":0.0471,"brier":0.3928,"over95":0.0049,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0007,"conf":0.0908,"acc":0.0511},{"lo":0.1,"w":0.0074,"conf":0.1552,"acc":0.1275},{"lo":0.2,"w":0.0164,"conf":0.2571,"acc":0.1647},{"lo":0.3,"w":0.0302,"conf":0.3557,"acc":0.2335},{"lo":0.4,"w":0.0488,"conf":0.4538,"acc":0.3905},{"lo":0.5,"w":0.0689,"conf":0.5529,"acc":0.5073},{"lo":0.6,"w":0.0853,"conf":0.651,"acc":0.5212},{"lo":0.7,"w":0.1592,"conf":0.7577,"acc":0.6701},{"lo":0.8,"w":0.3135,"conf":0.848,"acc":0.8283},{"lo":0.9,"w":0.2696,"conf":0.9374,"acc":0.9216}],"areas":{"knowledge":{"ece":0.0515,"acc":0.6931,"conf":0.6804},"language":{"ece":0.0254,"acc":0.8154,"conf":0.8204},"retrieval":{"ece":0.2449,"acc":0.6096,"conf":0.8545},"tools":{"ece":0.0472,"acc":0.9386,"conf":0.8952},"arts":{"ece":0.1508,"acc":0.59,"conf":0.7408}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":108.5,"p95":1114.7,"mean":358.6},"results":{"1":{"score":0.9799,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":328.5},"2":{"score":0.6359,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":5679.8},"3":{"score":0.8504,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7907,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":294.3},"5":{"score":0.9116,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":502.5},"6":{"score":0.8,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":741.6},"9":{"score":0.625,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.031,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":503.4},"11":{"score":0.7356,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7316,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":79.4},"20":{"score":0.9026,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":79.3},"21":{"score":0.6381,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":71.8},"22":{"score":0.285,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":2664.9},"23":{"score":0.6408,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":145.4},"24":{"score":0.863,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":72.3},"25":{"score":0.4898,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9882,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":71.8},"27":{"score":0.9744,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":71.8},"28":{"score":0.8366,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":71.8},"29":{"score":0.9202,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":100.1},"30":{"score":0.8256,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":82.8},"31":{"score":0.2234,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":241.3},"32":{"score":0.6423,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2933,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":877.4},"36":{"score":0.4359,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5311,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":108.5},"38":{"score":0.1883,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2773.0},"39":{"score":0.9087,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5107,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":71.8,"tracks":[{"track":"A · Arabic","score":0.5099,"headline":false},{"track":"A · English","score":0.5107,"headline":true},{"track":"C · Arabic pairs","score":0.92,"headline":false},{"track":"C · English pairs","score":0.985,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.732,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":80.1},"42":{"score":0.8285,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":207.7},"43":{"score":0.793,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.746,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":79.5},"45":{"score":0.1477,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2021,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":208.1},"50":{"score":0.4791,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":241.8},"56":{"score":0.875,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1112.6,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.6964,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":82.6,"edition_added":"v0.2","jev":0.827},"58":{"score":0.7621,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":79.6,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.7689,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":240.9,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.7386,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.7694,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":208.3,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.7642,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":125.1,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":67.22,"balanced_skill":57.33,"breadth_skill":56.2},"frozen_scores":{"balanced_raw":49.81,"balanced_skill":39.24,"breadth_skill":33.04},"categories":[{"id":"knowledge","raw":0.5591,"skill":0.4427,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.7226,"skill":0.6038,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.7176,"skill":0.6308,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.7826,"skill":0.7557,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.5412,"skill":0.3834,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9799,"skill":0.9729,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6359,"skill":0.5796,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.8,"skill":0.5285,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7864,"headline":false},{"track":"RouterBench-5shot","score":0.8136,"headline":false}]},"11":{"raw":0.7356,"skill":0.6176,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.9059,"skill":0.8117,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6381,"skill":0.2763,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.2785,"skill":0.2729,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.6263,"skill":0.2525,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8651,"skill":0.8201,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4898,"skill":0.3197,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.8256,"skill":0.7881,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.837,"headline":false},{"track":"GSM8K-10choice","score":0.8143,"headline":false}]},"31":{"raw":0.2234,"skill":0.1542,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4359,"skill":0.3619,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.5311,"skill":0.4118,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.5107,"skill":0.3705,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5099,"headline":false},{"track":"A · English","score":0.5107,"headline":true},{"track":"C · Arabic pairs","score":0.92,"headline":false},{"track":"C · English pairs","score":0.985,"headline":false}]},"41":{"raw":0.732,"skill":0.5981,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.793,"skill":0.6715,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.746,"skill":0.492,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4791,"skill":0.244,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.875,"skill":0.75,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.6964,"skill":0.6585,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.7621,"skill":0.6552,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.7689,"skill":0.6919,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.7386,"skill":0.6732,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.7694,"skill":0.5219,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.7642,"skill":0.5284,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.6423,"skill":0.4313,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2933,"skill":0.2839,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1477,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.7316,"skill":0.598,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.8366,"skill":0.6732,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.9202,"skill":0.8936,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.1883,"skill":0.1623,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.9087,"skill":0.8657,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.8285,"skill":0.6663,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.7907,"skill":0.788,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.9116,"skill":0.9111,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.8504,"skill":0.8475,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.625,"skill":0.625,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.1916,"skill":0.1916,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"decider2.serve_chat_adaptive over serve_chat_lowmem as board (graph server serve_chat OOMs on 96 GB); one sequential HTTP client; card venv torch 2.14/transformers 5.17 instead of the job's pytorch 2.10 image","new_in_edition":true},{"engine":"decider-chat-qwen3.6-27b","name":"Decider chat · Qwen3.6-27B","long_name":"Decider chat · Qwen3.6-27B","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/Mapika/decider","base_model":"Qwen/Qwen3.6-27B","base_url":"https://huggingface.co/Qwen/Qwen3.6-27B","weights_repo":null,"served_checkpoint":"Qwen/Qwen3.6-27B"},"completed":120340,"counts":{"ok":119898},"lifted":false,"calibration":{"acc":0.6968,"conf":0.6848,"ece":0.0209,"brier":0.3998,"over95":0.001,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0178,"conf":0.0629,"acc":0.1578},{"lo":0.1,"w":0.0226,"conf":0.1531,"acc":0.1685},{"lo":0.2,"w":0.0328,"conf":0.2495,"acc":0.2453},{"lo":0.3,"w":0.05,"conf":0.3558,"acc":0.3247},{"lo":0.4,"w":0.0754,"conf":0.4503,"acc":0.4396},{"lo":0.5,"w":0.1335,"conf":0.5491,"acc":0.5344},{"lo":0.6,"w":0.133,"conf":0.6521,"acc":0.6622},{"lo":0.7,"w":0.1458,"conf":0.7518,"acc":0.7561},{"lo":0.8,"w":0.1809,"conf":0.8513,"acc":0.8858},{"lo":0.9,"w":0.2083,"conf":0.9442,"acc":0.9741}],"areas":{"knowledge":{"ece":0.0329,"acc":0.6483,"conf":0.6508},"language":{"ece":0.0457,"acc":0.8102,"conf":0.7664},"retrieval":{"ece":0.0804,"acc":0.5994,"conf":0.6328},"tools":{"ece":0.0662,"acc":0.9059,"conf":0.8434},"arts":{"ece":0.0559,"acc":0.5517,"conf":0.5591}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":83.6,"p95":965.6,"mean":284.0},"results":{"1":{"score":0.9752,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":272.7},"2":{"score":0.6438,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":2975.6},"3":{"score":0.7972,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.751,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":234.8},"5":{"score":0.8506,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":372.5},"6":{"score":0.7991,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":505.3},"9":{"score":0.4432,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.1759,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":373.5},"11":{"score":0.74,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7459,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":63.4},"20":{"score":0.8764,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":63.3},"21":{"score":0.6084,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":57.0},"22":{"score":0.185,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1814.2},"23":{"score":0.6332,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":122.9},"24":{"score":0.8521,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":57.9},"25":{"score":0.5459,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9882,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":57.3},"27":{"score":0.9727,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":57.0},"28":{"score":0.8319,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":56.9},"29":{"score":0.9542,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":68.0},"30":{"score":0.5796,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":66.6},"31":{"score":0.1762,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":186.2},"32":{"score":0.6263,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3479,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":899.2},"36":{"score":0.4665,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4767,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":83.7},"38":{"score":0.2444,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2353.8},"39":{"score":0.8183,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.458,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":56.8,"tracks":[{"track":"A · Arabic","score":0.4615,"headline":false},{"track":"A · English","score":0.458,"headline":true},{"track":"C · Arabic pairs","score":0.8,"headline":false},{"track":"C · English pairs","score":0.975,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.5839,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":63.5},"42":{"score":0.8203,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":185.8},"43":{"score":0.6404,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6634,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":63.6},"45":{"score":0.1158,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.186,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":145.7},"50":{"score":0.4087,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":185.9},"56":{"score":0.6755,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":964.7,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.6287,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":66.5,"edition_added":"v0.2","jev":0.827},"58":{"score":0.7273,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":64.5,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.785,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":184.9,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.7273,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.7503,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":165.3,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.68,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":100.8,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":63.06,"balanced_skill":51.35,"breadth_skill":50.04},"frozen_scores":{"balanced_raw":46.14,"balanced_skill":33.91,"breadth_skill":28.72},"categories":[{"id":"knowledge","raw":0.5051,"skill":0.3699,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.7032,"skill":0.5712,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.6536,"skill":0.5223,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.7405,"skill":0.7142,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.5202,"skill":0.3506,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9752,"skill":0.9665,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6438,"skill":0.5886,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7991,"skill":0.5262,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.786,"headline":false},{"track":"RouterBench-5shot","score":0.8122,"headline":false}]},"11":{"raw":0.74,"skill":0.624,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.8784,"skill":0.7569,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6084,"skill":0.2169,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.1812,"skill":0.1748,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.6339,"skill":0.2678,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8592,"skill":0.8123,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5459,"skill":0.3946,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.5796,"skill":0.4928,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.6391,"headline":false},{"track":"GSM8K-10choice","score":0.5201,"headline":false}]},"31":{"raw":0.1762,"skill":0.1028,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4665,"skill":0.3964,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.4767,"skill":0.3436,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.458,"skill":0.3027,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4615,"headline":false},{"track":"A · English","score":0.458,"headline":true},{"track":"C · Arabic pairs","score":0.8,"headline":false},{"track":"C · English pairs","score":0.975,"headline":false}]},"41":{"raw":0.5839,"skill":0.3759,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.6404,"skill":0.4294,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6634,"skill":0.3268,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4087,"skill":0.1418,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.6755,"skill":0.351,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.6287,"skill":0.5824,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.7273,"skill":0.6047,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.785,"skill":0.7133,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.7273,"skill":0.6591,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.7503,"skill":0.4823,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.68,"skill":0.36,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.6263,"skill":0.4059,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.3479,"skill":0.3392,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1158,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.7459,"skill":0.6194,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.8319,"skill":0.6638,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.9542,"skill":0.9389,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.2444,"skill":0.2202,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8183,"skill":0.7328,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.8203,"skill":0.6504,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.751,"skill":0.7478,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.8506,"skill":0.8497,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.7972,"skill":0.7933,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.4432,"skill":0.4432,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.256,"skill":0.256,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author decider.serve v1.2.1 chat layout (bf16, graphs), raised admission limits as board; one sequential HTTP client (engine-launched server, not attached)"},{"engine":"decision-eos-0.8b","name":"Decision 1.0 Eos","long_name":"Decision 1.0 Eos 0.8B","variant":null,"meta":{"kind":"head / adapter","technique":"autoregressive","technique_name":"autoregressive","served_params":873438784,"params_estimated":false,"params_from_name":false,"trained_bytes":1509034472,"model_url":"https://huggingface.co/llm-semantic-router/Decision-1.0-Eos-0.8B","code_url":null,"base_model":"Qwen/Qwen3.5-0.8B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B-Base","weights_repo":"llm-semantic-router/Decision-1.0-Eos-0.8B","served_checkpoint":null},"completed":120340,"counts":{"ok":119896,"unsupported":2},"lifted":false,"calibration":{"acc":0.4799,"conf":0.5532,"ece":0.0833,"brier":0.6241,"over95":0.0052,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0657,"conf":0.0421,"acc":0.0519},{"lo":0.1,"w":0.0326,"conf":0.1572,"acc":0.2009},{"lo":0.2,"w":0.046,"conf":0.2569,"acc":0.2437},{"lo":0.3,"w":0.1082,"conf":0.3507,"acc":0.3546},{"lo":0.4,"w":0.1115,"conf":0.4492,"acc":0.4715},{"lo":0.5,"w":0.2028,"conf":0.5477,"acc":0.4937},{"lo":0.6,"w":0.1497,"conf":0.6472,"acc":0.5262},{"lo":0.7,"w":0.1084,"conf":0.7474,"acc":0.5391},{"lo":0.8,"w":0.0951,"conf":0.8494,"acc":0.6787},{"lo":0.9,"w":0.0799,"conf":0.9473,"acc":0.8244}],"areas":{"knowledge":{"ece":0.0899,"acc":0.4231,"conf":0.5069},"language":{"ece":0.1565,"acc":0.4842,"conf":0.6394},"retrieval":{"ece":0.0716,"acc":0.6163,"conf":0.561},"tools":{"ece":0.0683,"acc":0.6418,"conf":0.6053},"arts":{"ece":0.0799,"acc":0.3908,"conf":0.4625}}},"gaps":{"share":0.0,"reasons":[{"reason":"prompt longer than its 16,384-token context window","share":0.0}]},"latency":{"median":39.7,"p95":84.3,"mean":94.8},"results":{"1":{"score":0.5885,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":41.6},"2":{"score":0.5563,"jev":0.653,"metric":"nDCG@10","requests":685,"answered":684,"unsupported":1,"errors":0,"median_ms":232.1},"3":{"score":0.3661,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.6479,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":42.8},"5":{"score":0.6651,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":52.1},"6":{"score":0.7255,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":62.0},"9":{"score":0.0227,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6296,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":40.5},"11":{"score":0.436,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.384,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":39.3},"20":{"score":0.4934,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39.2},"21":{"score":0.5217,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":38.7},"22":{"score":0.0215,"jev":0.1811,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":109.7},"23":{"score":0.546,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39.5},"24":{"score":0.4345,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":39.1},"25":{"score":0.2449,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.8106,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":39.0},"27":{"score":0.6382,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":38.9},"28":{"score":0.5185,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":38.7},"29":{"score":0.4663,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":39.5},"30":{"score":0.2494,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":39.3},"31":{"score":0.0844,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":40.9},"32":{"score":0.5199,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0121,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":61.2},"36":{"score":0.2557,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3455,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39.9},"38":{"score":0.0553,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":334.5},"39":{"score":0.6282,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2388,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":38.8,"tracks":[{"track":"A · Arabic","score":0.2578,"headline":false},{"track":"A · English","score":0.2388,"headline":true},{"track":"C · Arabic pairs","score":0.345,"headline":false},{"track":"C · English pairs","score":0.325,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4314,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":39.2},"42":{"score":0.6056,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":39.8},"43":{"score":0.4123,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5184,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39.1},"45":{"score":0.1497,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2365,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":40.3},"50":{"score":0.386,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":40.9},"56":{"score":0.5,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":83.3,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.1992,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":39.5,"edition_added":"v0.2","jev":0.827},"58":{"score":0.4494,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":39.2,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.4496,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":40.2,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.3731,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.0241,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":40.6,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5405,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":39.8,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":37.81,"balanced_skill":18.41,"breadth_skill":16.56},"frozen_scores":{"balanced_raw":30.2,"balanced_skill":10.78,"breadth_skill":9.09},"categories":[{"id":"knowledge","raw":0.2823,"skill":0.0737,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.3832,"skill":0.1459,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.4952,"skill":0.3057,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.4025,"skill":0.321,"coverage":0.9997,"pending":0,"n":5},{"id":"arts","raw":0.334,"skill":0.0747,"coverage":0.9999,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.5885,"skill":0.4446,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.5555,"skill":0.4866,"coverage":0.9985,"random":0.1341,"tracks":[]},"6":{"raw":0.7256,"skill":0.3663,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.6582,"headline":false},{"track":"RouterBench-5shot","score":0.7929,"headline":false}]},"11":{"raw":0.436,"skill":0.1844,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.4925,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5217,"skill":0.0434,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0214,"skill":0.0138,"coverage":0.9995,"random":0.0078,"tracks":[]},"23":{"raw":0.5452,"skill":0.0905,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4437,"skill":0.2582,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2449,"skill":0.0,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.2494,"skill":0.0904,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.3192,"headline":false},{"track":"GSM8K-10choice","score":0.1797,"headline":false}]},"31":{"raw":0.0844,"skill":0.0028,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.2557,"skill":0.158,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.3455,"skill":0.179,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.2388,"skill":0.0207,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2578,"headline":false},{"track":"A · English","score":0.2388,"headline":true},{"track":"C · Arabic pairs","score":0.345,"headline":false},{"track":"C · English pairs","score":0.325,"headline":false}]},"41":{"raw":0.4314,"skill":0.1471,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4123,"skill":0.0675,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5184,"skill":0.0368,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.386,"skill":0.1089,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.1992,"skill":0.0993,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.4494,"skill":0.2019,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.4496,"skill":0.2661,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.3731,"skill":0.2164,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.0241,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5405,"skill":0.081,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5199,"skill":0.2367,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0121,"skill":0.0,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1497,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.384,"skill":0.0773,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5185,"skill":0.037,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.4663,"skill":0.2884,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0553,"skill":0.0251,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.6282,"skill":0.4532,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.6056,"skill":0.2327,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.6479,"skill":0.6434,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.6651,"skill":0.6631,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.3661,"skill":0.3539,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0227,"skill":0.0227,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.054,"skill":0.054,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.7599,"pending":0.24,"latency_rows":750,"latency_path":"author DecisionModel.decide (bf16, FLA gated-delta kernels, questions of a row batched at physical batch 8), venv-decision torch 2.13.0+cu130 / transformers 5.17.0 / fla 0.5.2; same as board (no causal-conv1d CUDA kernel, as on the board)"},{"engine":"decision-kai-7185f514","name":"Decision 1.0 Kai","long_name":"Decision 1.0 Kai","variant":null,"meta":{"kind":"head / adapter","technique":"encoder","technique_name":"encoder","served_params":307797035,"params_estimated":true,"params_from_name":false,"trained_bytes":2287687932,"model_url":"https://huggingface.co/llm-semantic-router/Decision-1.0-Kai","code_url":null,"base_model":"jhu-clsp/mmBERT-base","base_url":"https://huggingface.co/jhu-clsp/mmBERT-base","weights_repo":"llm-semantic-router/Decision-1.0-Kai","served_checkpoint":null},"completed":120340,"counts":{"ok":119918,"unsupported":319},"lifted":true,"calibration":{"acc":0.3579,"conf":0.5421,"ece":0.1851,"brier":0.7572,"over95":0.0037,"n":35090,"benchmarks":32,"rel":[{"lo":0.0,"w":0.049,"conf":0.0757,"acc":0.0629},{"lo":0.1,"w":0.0376,"conf":0.1449,"acc":0.1564},{"lo":0.2,"w":0.0639,"conf":0.258,"acc":0.1166},{"lo":0.3,"w":0.0884,"conf":0.3503,"acc":0.2687},{"lo":0.4,"w":0.1358,"conf":0.4521,"acc":0.3118},{"lo":0.5,"w":0.2279,"conf":0.5472,"acc":0.4418},{"lo":0.6,"w":0.1515,"conf":0.6454,"acc":0.4314},{"lo":0.7,"w":0.1061,"conf":0.7507,"acc":0.392},{"lo":0.8,"w":0.0983,"conf":0.8473,"acc":0.4744},{"lo":0.9,"w":0.0414,"conf":0.9317,"acc":0.5086}],"areas":{"knowledge":{"ece":0.205,"acc":0.3113,"conf":0.5163},"language":{"ece":0.1997,"acc":0.4252,"conf":0.6249},"retrieval":{"ece":0.1161,"acc":0.4696,"conf":0.4952},"tools":{"ece":0.3039,"acc":0.2193,"conf":0.5155},"arts":{"ece":0.1702,"acc":0.333,"conf":0.5031}}},"gaps":{"share":0.0027,"reasons":[{"reason":"ran out of GPU memory on the longest prompts","share":0.0026},{"reason":"prompt longer than its 8,192-token context window","share":0.0}]},"latency":{"median":30.4,"p95":106.8,"mean":42.0},"results":{"1":{"score":0.3884,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":32.5},"2":{"score":0.3265,"jev":0.6535,"metric":"nDCG@10","requests":685,"answered":680,"unsupported":5,"errors":0,"median_ms":null},"3":{"score":0.1594,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.4073,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":32.1},"5":{"score":0.0569,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":34.3},"6":{"score":0.6674,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":82.4},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4846,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":33.5},"11":{"score":0.3662,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.2428,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":29.8},"20":{"score":0.3986,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":29.9},"21":{"score":0.4574,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":29.5},"22":{"score":0.0015,"jev":0.1812,"metric":"accuracy","requests":2000,"answered":1998,"unsupported":2,"errors":0,"median_ms":165.5},"23":{"score":0.4934,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":29.9},"24":{"score":0.3106,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":29.8},"25":{"score":0.2296,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.487,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":29.7},"27":{"score":0.3183,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":29.5},"28":{"score":0.4917,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":29.7},"29":{"score":0.3404,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":30.3},"30":{"score":0.2915,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":30.0},"31":{"score":0.0684,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":31.3},"32":{"score":0.4521,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0024,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":57.7},"36":{"score":0.1385,"jev":0.4765,"metric":"nDCG@10","requests":220,"answered":219,"unsupported":1,"errors":0,"median_ms":null},"37":{"score":0.2045,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":30.3},"38":{"score":0.0383,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":155.3},"39":{"score":0.3695,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.0087,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":29.7,"tracks":[{"track":"A · Arabic","score":0.0394,"headline":false},{"track":"A · English","score":0.0087,"headline":true},{"track":"C · Arabic pairs","score":0.29,"headline":false},{"track":"C · English pairs","score":0.21,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.2081,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":29.9},"42":{"score":0.2976,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":30.7},"43":{"score":0.393,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.4978,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":29.7},"45":{"score":0.122,"jev":0.204,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":null},"48":{"score":0.4344,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":31.3},"50":{"score":0.3837,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":32.1},"56":{"score":0.499,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":69.3,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.1385,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":30.1,"edition_added":"v0.2","jev":0.827},"58":{"score":0.3497,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":30.0,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.3757,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":31.7,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.2727,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.5177,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":31.5,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5212,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":30.7,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":27.37,"balanced_skill":6.52,"breadth_skill":6.04},"frozen_scores":{"balanced_raw":23.95,"balanced_skill":4.46,"breadth_skill":3.95},"categories":[{"id":"knowledge","raw":0.2512,"skill":0.0383,"coverage":0.9998,"pending":0,"n":10},{"id":"language","raw":0.2921,"skill":0.0291,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.295,"skill":0.0932,"coverage":0.9992,"pending":0,"n":6},{"id":"tools","raw":0.2513,"skill":0.1423,"coverage":0.9986,"pending":0,"n":5},{"id":"arts","raw":0.2824,"skill":0.0308,"coverage":0.9999,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.3884,"skill":0.1744,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.3241,"skill":0.2195,"coverage":0.9927,"random":0.1341,"tracks":[]},"6":{"raw":0.6675,"skill":0.2195,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.6322,"headline":false},{"track":"RouterBench-5shot","score":0.7028,"headline":false}]},"11":{"raw":0.3662,"skill":0.0834,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.4047,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.4574,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0019,"skill":0.0,"coverage":0.999,"random":0.0078,"tracks":[]},"23":{"raw":0.5127,"skill":0.0254,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.312,"skill":0.0826,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2296,"skill":0.0,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.2915,"skill":0.142,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.3632,"headline":false},{"track":"GSM8K-10choice","score":0.2199,"headline":false}]},"31":{"raw":0.0684,"skill":0.0,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.1379,"skill":0.0247,"coverage":0.9955,"random":0.116,"tracks":[]},"37":{"raw":0.2045,"skill":0.0022,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.0087,"skill":0.0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0394,"headline":false},{"track":"A · English","score":0.0087,"headline":true},{"track":"C · Arabic pairs","score":0.29,"headline":false},{"track":"C · English pairs","score":0.21,"headline":false}]},"41":{"raw":0.2081,"skill":0.0,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.393,"skill":0.0369,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.4978,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3837,"skill":0.1055,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.499,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.1385,"skill":0.031,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.3497,"skill":0.0574,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.3757,"skill":0.1676,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.2727,"skill":0.0909,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.5177,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5212,"skill":0.0424,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.4521,"skill":0.1289,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0024,"skill":0.0,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1218,"skill":0.0,"coverage":0.998,"random":0.1641,"tracks":[]},"12":{"raw":0.2428,"skill":0.0,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.4917,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.3404,"skill":0.1205,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0383,"skill":0.0075,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.3695,"skill":0.0727,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.2976,"skill":0.0,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.4073,"skill":0.3997,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.0569,"skill":0.0512,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.1594,"skill":0.1432,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.7595,"pending":0.24,"latency_rows":749,"latency_path":"author encoder DecisionModel (Decision 1.0 decision-kai), predict_1k cap lifted (uncapped engine), venv-decision-legacy torch 2.13.0+cu130 / transformers 4.57.6"},{"engine":"decision-lex","name":"Decision 1.0 Lex","long_name":"Decision 1.0 Lex","variant":null,"meta":{"kind":"head / adapter","technique":"encoder","technique_name":"encoder","served_params":307797035,"params_estimated":true,"params_from_name":false,"trained_bytes":2287687932,"model_url":"https://huggingface.co/llm-semantic-router/decision-1.0-lex","code_url":null,"base_model":"jhu-clsp/mmBERT-base","base_url":"https://huggingface.co/jhu-clsp/mmBERT-base","weights_repo":"llm-semantic-router/decision-1.0-lex","served_checkpoint":null},"completed":120340,"counts":{"ok":120014,"unsupported":326},"lifted":true,"calibration":{"acc":0.3304,"conf":0.4733,"ece":0.1688,"brier":0.7598,"over95":0.0005,"n":72594,"benchmarks":32,"rel":[{"lo":0.0,"w":0.1389,"conf":0.0454,"acc":0.1321},{"lo":0.1,"w":0.0287,"conf":0.1415,"acc":0.1728},{"lo":0.2,"w":0.0481,"conf":0.2734,"acc":0.1954},{"lo":0.3,"w":0.1292,"conf":0.3543,"acc":0.2983},{"lo":0.4,"w":0.1236,"conf":0.4453,"acc":0.3381},{"lo":0.5,"w":0.2062,"conf":0.5478,"acc":0.4438},{"lo":0.6,"w":0.1487,"conf":0.6462,"acc":0.4167},{"lo":0.7,"w":0.1042,"conf":0.7466,"acc":0.4101},{"lo":0.8,"w":0.0572,"conf":0.8426,"acc":0.3539},{"lo":0.9,"w":0.0152,"conf":0.9203,"acc":0.0621}],"areas":{"knowledge":{"ece":0.136,"acc":0.2964,"conf":0.4324},"language":{"ece":0.2495,"acc":0.3387,"conf":0.5875},"retrieval":{"ece":0.1707,"acc":0.39,"conf":0.3303},"tools":{"ece":0.2023,"acc":0.2976,"conf":0.4796},"arts":{"ece":0.1076,"acc":0.3625,"conf":0.4685}}},"gaps":{"share":0.0027,"reasons":[{"reason":"ran out of GPU memory on the longest prompts","share":0.0027},{"reason":"prompt longer than its 8,192-token context window","share":0.0001}]},"latency":{"median":29.8,"p95":106.0,"mean":41.3},"results":{"1":{"score":0.3872,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":32.0},"2":{"score":0.2269,"jev":0.6535,"metric":"nDCG@10","requests":685,"answered":680,"unsupported":5,"errors":0,"median_ms":null},"3":{"score":0.1398,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.2356,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":31.7},"5":{"score":0.1121,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":33.9},"6":{"score":0.593,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":83.0},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4106,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":33.1},"11":{"score":0.2011,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.2256,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":29.4},"20":{"score":0.4798,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":29.2},"21":{"score":0.4871,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":29.0},"22":{"score":0.002,"jev":0.1812,"metric":"accuracy","requests":2000,"answered":1998,"unsupported":2,"errors":0,"median_ms":162.6},"23":{"score":0.512,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":29.3},"24":{"score":0.3017,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":29.3},"25":{"score":0.2602,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.476,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":29.0},"27":{"score":0.3038,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":29.1},"28":{"score":0.5067,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":29.1},"29":{"score":0.3331,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":29.6},"30":{"score":0.2343,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":29.6},"31":{"score":0.0652,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":30.6},"32":{"score":0.3896,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0006,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":57.1},"36":{"score":0.1253,"jev":0.4765,"metric":"nDCG@10","requests":220,"answered":219,"unsupported":1,"errors":0,"median_ms":null},"37":{"score":0.1929,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":30.0},"38":{"score":0.031,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":153.0},"39":{"score":0.2178,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.1896,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":29.0,"tracks":[{"track":"A · Arabic","score":0.2054,"headline":false},{"track":"A · English","score":0.1896,"headline":true},{"track":"C · Arabic pairs","score":0.485,"headline":false},{"track":"C · English pairs","score":0.335,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4039,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":29.4},"42":{"score":0.3026,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":30.0},"43":{"score":0.3684,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.4838,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":29.3},"45":{"score":0.148,"jev":0.204,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":null},"48":{"score":0.4767,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":30.8},"50":{"score":0.3246,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":31.4},"56":{"score":0.4975,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":68.7,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.145,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":29.5,"edition_added":"v0.2","jev":0.827},"58":{"score":0.3332,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":29.3,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.3445,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":30.6,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.2955,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.4913,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":31.1,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.501,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":30.1,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":26.12,"balanced_skill":4.54,"breadth_skill":4.21},"frozen_scores":{"balanced_raw":23.73,"balanced_skill":2.49,"breadth_skill":2.3},"categories":[{"id":"knowledge","raw":0.2412,"skill":0.0188,"coverage":0.9998,"pending":0,"n":10},{"id":"language","raw":0.2939,"skill":0.0241,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.2664,"skill":0.0625,"coverage":0.9992,"pending":0,"n":6},{"id":"tools","raw":0.2226,"skill":0.1086,"coverage":0.9986,"pending":0,"n":5},{"id":"arts","raw":0.2882,"skill":0.0193,"coverage":0.9999,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.3872,"skill":0.1728,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.2253,"skill":0.1053,"coverage":0.9927,"random":0.1341,"tracks":[]},"6":{"raw":0.593,"skill":0.0459,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.5453,"headline":false},{"track":"RouterBench-5shot","score":0.6407,"headline":false}]},"11":{"raw":0.2011,"skill":0.0,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.48,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.4871,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0012,"skill":0.0,"coverage":0.999,"random":0.0078,"tracks":[]},"23":{"raw":0.4868,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3079,"skill":0.0772,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2602,"skill":0.0136,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.2343,"skill":0.071,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.2964,"headline":false},{"track":"GSM8K-10choice","score":0.1721,"headline":false}]},"31":{"raw":0.0652,"skill":0.0,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.1247,"skill":0.0098,"coverage":0.9955,"random":0.116,"tracks":[]},"37":{"raw":0.1929,"skill":0.0,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.1896,"skill":0.0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2054,"headline":false},{"track":"A · English","score":0.1896,"headline":true},{"track":"C · Arabic pairs","score":0.485,"headline":false},{"track":"C · English pairs","score":0.335,"headline":false}]},"41":{"raw":0.4039,"skill":0.1059,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3684,"skill":0.0,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.4838,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3246,"skill":0.0197,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.4975,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.145,"skill":0.0384,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.3332,"skill":0.0335,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.3445,"skill":0.126,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.2955,"skill":0.1194,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.4913,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.501,"skill":0.002,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.3896,"skill":0.0296,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0006,"skill":0.0,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1477,"skill":0.0,"coverage":0.998,"random":0.1641,"tracks":[]},"12":{"raw":0.2256,"skill":0.0,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5067,"skill":0.0134,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.3331,"skill":0.1108,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.031,"skill":0.0,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.2178,"skill":0.0,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.3026,"skill":0.0,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.2356,"skill":0.2258,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.1121,"skill":0.1067,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.1398,"skill":0.1232,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.7595,"pending":0.24,"latency_rows":749,"latency_path":"author encoder DecisionModel (Decision 1.0 decision-lex), predict_1k cap lifted (uncapped engine), venv-decision-legacy torch 2.13.0+cu130 / transformers 4.57.6"},{"engine":"decision-lux-9b","name":"Decision 1.0 Lux","long_name":"Decision 1.0 Lux 9B","variant":null,"meta":{"kind":"head / adapter","technique":"autoregressive","technique_name":"autoregressive","served_params":9653104368,"params_estimated":false,"params_from_name":false,"trained_bytes":15890261552,"model_url":"https://huggingface.co/llm-semantic-router/Decision-1.0-Lux-9B","code_url":null,"base_model":"Qwen/Qwen3.5-9B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B-Base","weights_repo":"llm-semantic-router/Decision-1.0-Lux-9B","served_checkpoint":null},"completed":120340,"counts":{"ok":119898},"lifted":false,"calibration":{"acc":0.6636,"conf":0.7263,"ece":0.076,"brier":0.4397,"over95":0.0165,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0454,"conf":0.0528,"acc":0.171},{"lo":0.1,"w":0.0161,"conf":0.1402,"acc":0.1256},{"lo":0.2,"w":0.0165,"conf":0.253,"acc":0.3295},{"lo":0.3,"w":0.0403,"conf":0.3551,"acc":0.2816},{"lo":0.4,"w":0.0532,"conf":0.4524,"acc":0.3299},{"lo":0.5,"w":0.1179,"conf":0.5502,"acc":0.4866},{"lo":0.6,"w":0.109,"conf":0.6513,"acc":0.5688},{"lo":0.7,"w":0.1113,"conf":0.7501,"acc":0.6423},{"lo":0.8,"w":0.1229,"conf":0.8538,"acc":0.7504},{"lo":0.9,"w":0.3675,"conf":0.9655,"acc":0.9153}],"areas":{"knowledge":{"ece":0.0812,"acc":0.5976,"conf":0.6728},"language":{"ece":0.0727,"acc":0.7606,"conf":0.8315},"retrieval":{"ece":0.0754,"acc":0.6747,"conf":0.7411},"tools":{"ece":0.0283,"acc":0.8794,"conf":0.8951},"arts":{"ece":0.1166,"acc":0.4957,"conf":0.5416}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":51.0,"p95":347.5,"mean":114.1},"results":{"1":{"score":0.9427,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":77.1},"2":{"score":0.6368,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1103.1},"3":{"score":0.7402,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8065,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":108.7},"5":{"score":0.8842,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":197.6},"6":{"score":0.7836,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":279.4},"9":{"score":0.25,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4604,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":70.3},"11":{"score":0.7166,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6641,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":50.3},"20":{"score":0.686,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":50.1},"21":{"score":0.5613,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":50.2},"22":{"score":0.1945,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":464.6},"23":{"score":0.6168,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":51.0},"24":{"score":0.771,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":50.4},"25":{"score":0.4337,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.984,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":50.4},"27":{"score":0.9488,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":50.2},"28":{"score":0.7782,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":50.0},"29":{"score":0.8867,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":50.6},"30":{"score":0.5091,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":50.4},"31":{"score":0.136,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":76.1},"32":{"score":0.5838,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3303,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":253.9},"36":{"score":0.4141,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4602,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":51.0},"38":{"score":0.1627,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":742.5},"39":{"score":0.8181,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4282,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":50.7,"tracks":[{"track":"A · Arabic","score":0.4026,"headline":false},{"track":"A · English","score":0.4282,"headline":true},{"track":"C · Arabic pairs","score":0.585,"headline":false},{"track":"C · English pairs","score":0.86,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4393,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":50.4},"42":{"score":0.8076,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":51.7},"43":{"score":0.6228,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.645,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":50.4},"45":{"score":0.0958,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.1907,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":55.4},"50":{"score":0.4141,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":65.5},"56":{"score":0.592,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":177.7,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.5441,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":50.7,"edition_added":"v0.2","jev":0.827},"58":{"score":0.6755,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":50.5,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.4513,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":54.5,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.6231,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.6599,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":55.8,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.6825,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":51.4,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":57.43,"balanced_skill":43.49,"breadth_skill":42.33},"frozen_scores":{"balanced_raw":42.26,"balanced_skill":27.79,"breadth_skill":23.49},"categories":[{"id":"knowledge","raw":0.4561,"skill":0.309,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.6441,"skill":0.4797,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.6455,"skill":0.5002,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.6218,"skill":0.5719,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.47,"skill":0.2635,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9427,"skill":0.9227,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6368,"skill":0.5806,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7836,"skill":0.4876,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.777,"headline":false},{"track":"RouterBench-5shot","score":0.7902,"headline":false}]},"11":{"raw":0.7166,"skill":0.5902,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.6953,"skill":0.3906,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5613,"skill":0.1225,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.184,"skill":0.1777,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.6217,"skill":0.2434,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7806,"skill":0.7075,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4337,"skill":0.2449,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.5091,"skill":0.4134,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.63,"headline":false},{"track":"GSM8K-10choice","score":0.3882,"headline":false}]},"31":{"raw":0.136,"skill":0.059,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4141,"skill":0.3372,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.4602,"skill":0.3229,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.4282,"skill":0.2644,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4026,"headline":false},{"track":"A · English","score":0.4282,"headline":true},{"track":"C · Arabic pairs","score":0.585,"headline":false},{"track":"C · English pairs","score":0.86,"headline":false}]},"41":{"raw":0.4393,"skill":0.1589,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.6228,"skill":0.4015,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.645,"skill":0.29,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4141,"skill":0.1496,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.592,"skill":0.184,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.5441,"skill":0.4872,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6755,"skill":0.5296,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.4513,"skill":0.2684,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.6231,"skill":0.5289,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.6599,"skill":0.2948,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.6825,"skill":0.365,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5838,"skill":0.3383,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.3303,"skill":0.3214,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.0958,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.6641,"skill":0.4969,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.7782,"skill":0.5564,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.8867,"skill":0.8489,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.1627,"skill":0.1359,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8181,"skill":0.7325,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.8076,"skill":0.6257,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.8065,"skill":0.804,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.8842,"skill":0.8835,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.7402,"skill":0.7352,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.25,"skill":0.25,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.2372,"skill":0.2372,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author DecisionModel.decide (bf16, FLA gated-delta kernels with sm_120 autotune, questions batched in-row), input limit raised to the backbone window as on the board, venv-decision torch 2.13.0+cu130"},{"engine":"decision-nox-0bb83350","name":"Decision 1.0 Nox","long_name":"Decision 1.0 Nox","variant":null,"meta":{"kind":"head / adapter","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":8422078976,"model_url":"https://huggingface.co/llm-semantic-router/Decision-1.0-Nox","code_url":null,"base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"llm-semantic-router/Decision-1.0-Nox","served_checkpoint":null},"completed":120340,"counts":{"ok":120171},"lifted":false,"calibration":{"acc":0.6141,"conf":0.754,"ece":0.1399,"brier":0.5298,"over95":0.0489,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0151,"conf":0.0815,"acc":0.0522},{"lo":0.1,"w":0.0338,"conf":0.1409,"acc":0.0761},{"lo":0.2,"w":0.0178,"conf":0.2536,"acc":0.1859},{"lo":0.3,"w":0.0389,"conf":0.3575,"acc":0.2254},{"lo":0.4,"w":0.0589,"conf":0.4526,"acc":0.3511},{"lo":0.5,"w":0.1039,"conf":0.5463,"acc":0.4772},{"lo":0.6,"w":0.0958,"conf":0.6505,"acc":0.534},{"lo":0.7,"w":0.0947,"conf":0.7505,"acc":0.583},{"lo":0.8,"w":0.1156,"conf":0.8519,"acc":0.6398},{"lo":0.9,"w":0.4255,"conf":0.9737,"acc":0.8181}],"areas":{"knowledge":{"ece":0.1528,"acc":0.5364,"conf":0.6892},"language":{"ece":0.1125,"acc":0.6873,"conf":0.7963},"retrieval":{"ece":0.1394,"acc":0.7065,"conf":0.8436},"tools":{"ece":0.0827,"acc":0.872,"conf":0.932},"arts":{"ece":0.217,"acc":0.4249,"conf":0.6419}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":53.2,"p95":242.1,"mean":136.3},"results":{"1":{"score":0.8583,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":63.0},"2":{"score":0.6204,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":726.8},"3":{"score":0.7756,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8331,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":75.3},"5":{"score":0.853,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":136.8},"6":{"score":0.7346,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":183.5},"9":{"score":0.0455,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6422,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":53.6},"11":{"score":0.6408,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5978,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":52.5},"20":{"score":0.5136,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":52.2},"21":{"score":0.5084,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":52.1},"22":{"score":0.131,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":323.5},"23":{"score":0.592,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":52.4},"24":{"score":0.6054,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":52.4},"25":{"score":0.3112,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9541,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":52.8},"27":{"score":0.837,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":51.7},"28":{"score":0.6504,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":52.9},"29":{"score":0.7907,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":52.4},"30":{"score":0.4405,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":52.5},"31":{"score":0.074,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":54.6},"32":{"score":0.6037,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2788,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":167.4},"36":{"score":0.3459,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3851,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":53.2},"38":{"score":0.0729,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":486.0},"39":{"score":0.8133,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.378,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":52.7,"tracks":[{"track":"A · Arabic","score":0.3798,"headline":false},{"track":"A · English","score":0.378,"headline":true},{"track":"C · Arabic pairs","score":0.38,"headline":false},{"track":"C · English pairs","score":0.54,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4728,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":52.4},"42":{"score":0.7517,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":53.5},"43":{"score":0.4737,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6338,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":52.6},"45":{"score":0.1018,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.209,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":54.3},"50":{"score":0.3699,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":53.9},"56":{"score":0.5415,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":136.5,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.3669,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":52.6,"edition_added":"v0.2","jev":0.827},"58":{"score":0.5784,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":52.0,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.3801,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":53.6,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.4981,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.5094,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":54.5,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.6405,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":53.4,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":50.99,"balanced_skill":34.36,"breadth_skill":32.59},"frozen_scores":{"balanced_raw":36.62,"balanced_skill":19.4,"breadth_skill":16.01},"categories":[{"id":"knowledge","raw":0.3828,"skill":0.2141,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.5741,"skill":0.3659,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.6069,"skill":0.4431,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.5568,"skill":0.4957,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.3921,"skill":0.1439,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.8583,"skill":0.8087,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6204,"skill":0.5616,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7346,"skill":0.3805,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.6939,"headline":false},{"track":"RouterBench-5shot","score":0.7753,"headline":false}]},"11":{"raw":0.6408,"skill":0.4806,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.5184,"skill":0.0369,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5084,"skill":0.0167,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.1292,"skill":0.1224,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.6025,"skill":0.2051,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6157,"skill":0.4876,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3112,"skill":0.0816,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.4405,"skill":0.3306,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.5701,"headline":false},{"track":"GSM8K-10choice","score":0.3108,"headline":false}]},"31":{"raw":0.074,"skill":0.0,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.3459,"skill":0.26,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.3851,"skill":0.2287,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.378,"skill":0.1998,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3798,"headline":false},{"track":"A · English","score":0.378,"headline":true},{"track":"C · Arabic pairs","score":0.38,"headline":false},{"track":"C · English pairs","score":0.54,"headline":false}]},"41":{"raw":0.4728,"skill":0.2093,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4737,"skill":0.165,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6338,"skill":0.2676,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3699,"skill":0.0856,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5415,"skill":0.083,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.3669,"skill":0.2879,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.5784,"skill":0.3889,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.3801,"skill":0.1735,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.4981,"skill":0.3726,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.5094,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.6405,"skill":0.281,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.6037,"skill":0.37,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2788,"skill":0.2692,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1018,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.5978,"skill":0.3975,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.6504,"skill":0.3008,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.7907,"skill":0.7209,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0729,"skill":0.0432,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8133,"skill":0.7254,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7517,"skill":0.5169,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.8331,"skill":0.831,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.853,"skill":0.8521,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.7756,"skill":0.7713,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0455,"skill":0.0455,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.164,"skill":0.164,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author DecisionModel.from_pretrained(...).decide (bf16, FLA gated-delta kernels with sm_120 autotune instead of the gfx942 l2norm profile, questions batched in-row), venv-decision torch 2.13.0+cu130 / transformers 5.17.0 / fla 0.5.2; same as board"},{"engine":"decision-sol-0665a411","name":"Decision 1.0 Sol","long_name":"Decision 1.0 Sol","variant":null,"meta":{"kind":"head / adapter","technique":"autoregressive","technique_name":"autoregressive","served_params":2274069824,"params_estimated":false,"params_from_name":false,"trained_bytes":3772109600,"model_url":"https://huggingface.co/llm-semantic-router/Decision-1.0-Sol","code_url":null,"base_model":"Qwen/Qwen3.5-2B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-2B-Base","weights_repo":"llm-semantic-router/Decision-1.0-Sol","served_checkpoint":null},"completed":120340,"counts":{"ok":120226},"lifted":false,"calibration":{"acc":0.5439,"conf":0.6577,"ece":0.115,"brier":0.5828,"over95":0.0224,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0513,"conf":0.0521,"acc":0.0309},{"lo":0.1,"w":0.0128,"conf":0.1533,"acc":0.1019},{"lo":0.2,"w":0.0245,"conf":0.2609,"acc":0.2862},{"lo":0.3,"w":0.0651,"conf":0.3565,"acc":0.2785},{"lo":0.4,"w":0.0866,"conf":0.4484,"acc":0.3589},{"lo":0.5,"w":0.1641,"conf":0.5493,"acc":0.4576},{"lo":0.6,"w":0.1379,"conf":0.6483,"acc":0.5232},{"lo":0.7,"w":0.1114,"conf":0.7505,"acc":0.6188},{"lo":0.8,"w":0.1139,"conf":0.8513,"acc":0.671},{"lo":0.9,"w":0.2323,"conf":0.966,"acc":0.8272}],"areas":{"knowledge":{"ece":0.1133,"acc":0.4746,"conf":0.5874},"language":{"ece":0.1263,"acc":0.6165,"conf":0.7426},"retrieval":{"ece":0.1479,"acc":0.5832,"conf":0.729},"tools":{"ece":0.0632,"acc":0.7562,"conf":0.7977},"arts":{"ece":0.1113,"acc":0.4071,"conf":0.5183}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":38.1,"p95":101.0,"mean":53.6},"results":{"1":{"score":0.8855,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":38.9},"2":{"score":0.5445,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":312.0},"3":{"score":0.6201,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7705,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":39.8},"5":{"score":0.5523,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":67.6},"6":{"score":0.6963,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":82.1},"9":{"score":0.0114,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6149,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":39.2},"11":{"score":0.5014,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.4915,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":37.8},"20":{"score":0.5144,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":37.6},"21":{"score":0.5578,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":37.3},"22":{"score":0.0295,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":140.2},"23":{"score":0.557,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":38.0},"24":{"score":0.484,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":37.4},"25":{"score":0.2755,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.8948,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":37.4},"27":{"score":0.6877,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":37.3},"28":{"score":0.5864,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":37.3},"29":{"score":0.5838,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":37.7},"30":{"score":0.3256,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":37.7},"31":{"score":0.0648,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":38.6},"32":{"score":0.5532,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.1261,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":75.4},"36":{"score":0.2778,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3222,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":38.3},"38":{"score":0.1145,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":305.6},"39":{"score":0.6102,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2491,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":37.6,"tracks":[{"track":"A · Arabic","score":0.2974,"headline":false},{"track":"A · English","score":0.2491,"headline":true},{"track":"C · Arabic pairs","score":0.285,"headline":false},{"track":"C · English pairs","score":0.29,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4999,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":37.5},"42":{"score":0.6838,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":38.6},"43":{"score":0.3754,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5546,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":37.5},"45":{"score":0.1257,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.3362,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":38.8},"50":{"score":0.3168,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":38.6},"56":{"score":0.5015,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":81.0,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.2472,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":37.7,"edition_added":"v0.2","jev":0.827},"58":{"score":0.4837,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":37.6,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.4409,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":38.7,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.4432,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.6139,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":39.2,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.603,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":38.4,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":43.88,"balanced_skill":25.32,"breadth_skill":22.95},"frozen_scores":{"balanced_raw":32.27,"balanced_skill":13.8,"breadth_skill":11.35},"categories":[{"id":"knowledge","raw":0.3109,"skill":0.1149,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.4969,"skill":0.2535,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.5072,"skill":0.3271,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.5192,"skill":0.4611,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.3356,"skill":0.0818,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.8855,"skill":0.8454,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.5445,"skill":0.474,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.6963,"skill":0.286,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.6683,"headline":false},{"track":"RouterBench-5shot","score":0.7243,"headline":false}]},"11":{"raw":0.5014,"skill":0.2789,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.518,"skill":0.0359,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5578,"skill":0.1157,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0275,"skill":0.0199,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5527,"skill":0.1053,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4903,"skill":0.3205,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2755,"skill":0.034,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.3256,"skill":0.1913,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.4655,"headline":false},{"track":"GSM8K-10choice","score":0.1857,"headline":false}]},"31":{"raw":0.0648,"skill":0.0,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.2778,"skill":0.183,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.3222,"skill":0.1499,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.2491,"skill":0.034,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2974,"headline":false},{"track":"A · English","score":0.2491,"headline":true},{"track":"C · Arabic pairs","score":0.285,"headline":false},{"track":"C · English pairs","score":0.29,"headline":false}]},"41":{"raw":0.4999,"skill":0.2498,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3754,"skill":0.0091,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5546,"skill":0.1092,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3168,"skill":0.0085,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5015,"skill":0.003,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.2472,"skill":0.1533,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.4837,"skill":0.2516,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.4409,"skill":0.2545,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.4432,"skill":0.304,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.6139,"skill":0.1995,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.603,"skill":0.206,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5532,"skill":0.2897,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.1261,"skill":0.1145,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1257,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.4915,"skill":0.2383,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5864,"skill":0.1728,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.5838,"skill":0.4451,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.1145,"skill":0.0862,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.6102,"skill":0.4267,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.6838,"skill":0.3848,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.7705,"skill":0.7675,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.5523,"skill":0.5496,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.6201,"skill":0.6128,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0114,"skill":0.0114,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author DecisionModel.from_pretrained(...).decide (bf16, FLA gated-delta kernels with sm_120 autotune instead of the gfx942 l2norm profile, questions batched in-row), venv-decision torch 2.13.0+cu130 / transformers 5.17.0 / fla 0.5.2; same as board"},{"engine":"djev","name":"djev","long_name":"djev diffgemma","variant":null,"meta":{"kind":"inference technique","technique":"diffusion","technique_name":"diffusion","served_params":25823778864,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/Davipar/djev-dev","base_model":"google/diffusiongemma-26B-A4B-it","base_url":"https://huggingface.co/google/diffusiongemma-26B-A4B-it","weights_repo":null,"served_checkpoint":"google/diffusiongemma-26B-A4B-it"},"completed":120340,"counts":{"ok":119175,"unsupported":1165},"lifted":true,"calibration":{"acc":0.632,"conf":0.8445,"ece":0.2124,"brier":0.5655,"over95":0.0816,"n":29650,"benchmarks":32,"rel":[{"lo":0.0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0031,"conf":0.1634,"acc":0.0371},{"lo":0.2,"w":0.0141,"conf":0.258,"acc":0.1395},{"lo":0.3,"w":0.0258,"conf":0.3538,"acc":0.1978},{"lo":0.4,"w":0.0362,"conf":0.4563,"acc":0.2261},{"lo":0.5,"w":0.0682,"conf":0.549,"acc":0.4223},{"lo":0.6,"w":0.0757,"conf":0.6528,"acc":0.4607},{"lo":0.7,"w":0.0773,"conf":0.7524,"acc":0.4869},{"lo":0.8,"w":0.1148,"conf":0.8541,"acc":0.5321},{"lo":0.9,"w":0.5849,"conf":0.9774,"acc":0.7767}],"areas":{"knowledge":{"ece":0.2044,"acc":0.5853,"conf":0.7897},"language":{"ece":0.1444,"acc":0.7699,"conf":0.9143},"retrieval":{"ece":0.4174,"acc":0.4502,"conf":0.8677},"tools":{"ece":0.1155,"acc":0.7759,"conf":0.8833},"arts":{"ece":0.2516,"acc":0.5458,"conf":0.7974}}},"gaps":{"share":0.0097,"reasons":[{"reason":"more questions per request than its answer canvas holds","share":0.0097}]},"latency":{"median":84.4,"p95":211.2,"mean":108.8},"results":{"1":{"score":0.9292,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":96.3},"2":{"score":0.542,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":598.5},"3":{"score":0.7067,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.6944,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":120.7},"5":{"score":0.1352,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":145.4},"6":{"score":0.7988,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":204.4},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.1479,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":112.6},"11":{"score":0.7002,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5981,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":78.8},"20":{"score":0.9028,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":78.6},"21":{"score":0.6088,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":70.5},"22":{"score":0.15,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":169.1},"23":{"score":0.5942,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":85.6},"24":{"score":0.7943,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":77.2},"25":{"score":0.4745,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.984,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":74.8},"27":{"score":0.9522,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":76.4},"28":{"score":0.6938,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":70.3},"29":{"score":0.8386,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":80.5},"30":{"score":0.3669,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":78.6},"31":{"score":0.1402,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":99.8},"32":{"score":0.633,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2818,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":106.8},"36":{"score":0.3084,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5643,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":81.2},"38":{"score":0.2749,"jev":0.5291,"metric":"per-review F1","requests":1565,"answered":400,"unsupported":1165,"errors":0,"median_ms":null},"39":{"score":0.8438,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5967,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":70.1,"tracks":[{"track":"A · Arabic","score":0.2825,"headline":false},{"track":"A · English","score":0.5967,"headline":true},{"track":"C · Arabic pairs","score":0.765,"headline":false},{"track":"C · English pairs","score":0.96,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.6458,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":77.9},"42":{"score":0.7509,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":88.3},"43":{"score":0.4684,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6392,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":79.2},"45":{"score":0.1178,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2234,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":98.0},"50":{"score":0.4666,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":102.0},"56":{"score":0.8605,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":86.4,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.4404,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":79.2,"edition_added":"v0.2","jev":0.827},"58":{"score":0.6726,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":78.9,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.5449,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":96.4,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.6042,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.7306,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":97.4,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.6683,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":86.5,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":53.36,"balanced_skill":40.28,"breadth_skill":39.22},"frozen_scores":{"balanced_raw":42.95,"balanced_skill":29.73,"breadth_skill":25.32},"categories":[{"id":"knowledge","raw":0.4237,"skill":0.264,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.6505,"skill":0.4876,"coverage":0.9298,"pending":0,"n":10},{"id":"retrieval","raw":0.524,"skill":0.4177,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.5648,"skill":0.5168,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4777,"skill":0.3045,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9292,"skill":0.9044,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.542,"skill":0.4711,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7989,"skill":0.5257,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7854,"headline":false},{"track":"RouterBench-5shot","score":0.8123,"headline":false}]},"11":{"raw":0.7002,"skill":0.5664,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.9047,"skill":0.8095,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6088,"skill":0.2177,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.1425,"skill":0.1358,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5852,"skill":0.1704,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8013,"skill":0.735,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4745,"skill":0.2993,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.3669,"skill":0.2298,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.3988,"headline":false},{"track":"GSM8K-10choice","score":0.3351,"headline":false}]},"31":{"raw":0.1402,"skill":0.0635,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.3084,"skill":0.2177,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.5643,"skill":0.4534,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.5967,"skill":0.4811,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2825,"headline":false},{"track":"A · English","score":0.5967,"headline":true},{"track":"C · Arabic pairs","score":0.765,"headline":false},{"track":"C · English pairs","score":0.96,"headline":false}]},"41":{"raw":0.6458,"skill":0.4687,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4684,"skill":0.1566,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6392,"skill":0.2784,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4666,"skill":0.2258,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.8605,"skill":0.721,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.4404,"skill":0.3706,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6726,"skill":0.5254,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.5449,"skill":0.3932,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.6042,"skill":0.5052,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.7306,"skill":0.4414,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.6683,"skill":0.3366,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.633,"skill":0.4165,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2818,"skill":0.2723,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1178,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.5981,"skill":0.398,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.6938,"skill":0.3876,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.8386,"skill":0.7848,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0703,"skill":0.0405,"coverage":0.2556,"random":0.031,"tracks":[]},"39":{"raw":0.8438,"skill":0.7703,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7509,"skill":0.5154,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.6944,"skill":0.6905,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.1352,"skill":0.13,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.7067,"skill":0.701,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.1064,"skill":0.1064,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":744,"latency_path":"board DjevLong: djev in-process adapter over pinned vLLM 0.29.1rc1.dev347, prefix caching, max-model-len 262144","new_in_edition":true},{"engine":"gliner-base","name":"GLiNER 2.5 base","long_name":"GLiNER 2.5 base","variant":null,"meta":{"kind":"full fine-tune","technique":"GLi family","technique_name":"GLiNER2","served_params":193581591,"params_estimated":false,"params_from_name":false,"trained_bytes":774366564,"model_url":"https://huggingface.co/fastino/gliner2.5-base-v1","code_url":null,"base_model":null,"base_url":null,"weights_repo":"fastino/gliner2.5-base-v1","served_checkpoint":null},"completed":120340,"counts":{"ok":120331,"unsupported":9},"lifted":true,"calibration":{"acc":0.3476,"conf":0.7146,"ece":0.367,"brier":0.9595,"over95":0.1669,"n":72594,"benchmarks":32,"rel":[{"lo":0.0,"w":0.027,"conf":0.0695,"acc":0.0399},{"lo":0.1,"w":0.0237,"conf":0.1434,"acc":0.0624},{"lo":0.2,"w":0.0384,"conf":0.2599,"acc":0.1158},{"lo":0.3,"w":0.0535,"conf":0.3519,"acc":0.1868},{"lo":0.4,"w":0.0752,"conf":0.4511,"acc":0.2538},{"lo":0.5,"w":0.1161,"conf":0.5481,"acc":0.3745},{"lo":0.6,"w":0.0964,"conf":0.6486,"acc":0.3657},{"lo":0.7,"w":0.0901,"conf":0.7499,"acc":0.4141},{"lo":0.8,"w":0.1095,"conf":0.8516,"acc":0.4291},{"lo":0.9,"w":0.3702,"conf":0.9716,"acc":0.4011}],"areas":{"knowledge":{"ece":0.3672,"acc":0.3101,"conf":0.6772},"language":{"ece":0.465,"acc":0.384,"conf":0.8485},"retrieval":{"ece":0.2449,"acc":0.3619,"conf":0.6068},"tools":{"ece":0.408,"acc":0.2924,"conf":0.6968},"arts":{"ece":0.2664,"acc":0.3863,"conf":0.6528}}},"gaps":{"share":0.0001,"reasons":[{"reason":"prompt text with parentheses, which the native GLiNER prompt syntax cannot carry","share":0.0001}]},"latency":{"median":14.1,"p95":62.1,"mean":46.6},"results":{"1":{"score":0.3878,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":15.4},"2":{"score":0.2274,"jev":0.6529,"metric":"nDCG@10","requests":685,"answered":684,"unsupported":1,"errors":0,"median_ms":1327.5},"3":{"score":0.1555,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.2383,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":17.7},"5":{"score":0.2689,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":31.5},"6":{"score":0.6718,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":40.4},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.459,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":19.8},"11":{"score":0.2285,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.3048,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":13.3},"20":{"score":0.524,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":13.3},"21":{"score":0.4912,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":13.0},"22":{"score":0.036,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":358.3},"23":{"score":0.503,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.5},"24":{"score":0.3267,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":13.3},"25":{"score":0.25,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.4821,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2373,"unsupported":3,"errors":0,"median_ms":13.1},"27":{"score":0.2858,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":13.2},"28":{"score":0.5083,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":13.1},"29":{"score":0.3479,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":13.8},"30":{"score":0.2237,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":13.6},"31":{"score":0.0742,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":15.7},"32":{"score":0.3484,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0012,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":18.4},"36":{"score":0.1265,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.2005,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.2},"38":{"score":0.031,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":416.0},"39":{"score":0.7008,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2384,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":13.0,"tracks":[{"track":"A · Arabic","score":0.2591,"headline":false},{"track":"A · English","score":0.2384,"headline":true},{"track":"C · Arabic pairs","score":0.505,"headline":false},{"track":"C · English pairs","score":0.445,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3575,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":13.4},"42":{"score":0.3981,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":14.3},"43":{"score":0.4,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5164,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":13.3},"45":{"score":0.138,"jev":0.204,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":null},"48":{"score":0.4664,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":14.3},"50":{"score":0.3562,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":15.9},"56":{"score":0.5045,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":15.4,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.1408,"metric":"accuracy","requests":12032,"answered":12031,"unsupported":1,"errors":0,"median_ms":13.8,"edition_added":"v0.2","jev":0.827},"58":{"score":0.3361,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":13.4,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.3401,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":15.5,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.2652,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.4836,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":15.5,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5112,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":14.8,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":28.71,"balanced_skill":6.76,"breadth_skill":6.4},"frozen_scores":{"balanced_raw":25.05,"balanced_skill":3.87,"breadth_skill":3.48},"categories":[{"id":"knowledge","raw":0.2409,"skill":0.021,"coverage":0.9998,"pending":0,"n":10},{"id":"language","raw":0.3614,"skill":0.0748,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.2995,"skill":0.0966,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.2258,"skill":0.1116,"coverage":0.9997,"pending":0,"n":5},{"id":"arts","raw":0.3019,"skill":0.0313,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.3878,"skill":0.1736,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.2271,"skill":0.1074,"coverage":0.9985,"random":0.1341,"tracks":[]},"6":{"raw":0.6718,"skill":0.2289,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.6395,"headline":false},{"track":"RouterBench-5shot","score":0.7041,"headline":false}]},"11":{"raw":0.2285,"skill":0.0,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.5188,"skill":0.0377,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.4912,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0362,"skill":0.0287,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5059,"skill":0.0119,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3286,"skill":0.1047,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.25,"skill":0.0,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.2237,"skill":0.0608,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.3108,"headline":false},{"track":"GSM8K-10choice","score":0.1365,"headline":false}]},"31":{"raw":0.0742,"skill":0.0,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.1265,"skill":0.0119,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.2005,"skill":0.0,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.2384,"skill":0.0202,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2591,"headline":false},{"track":"A · English","score":0.2384,"headline":true},{"track":"C · Arabic pairs","score":0.505,"headline":false},{"track":"C · English pairs","score":0.445,"headline":false}]},"41":{"raw":0.3575,"skill":0.0363,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4,"skill":0.048,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5164,"skill":0.0328,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3562,"skill":0.0656,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5045,"skill":0.009,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.1408,"skill":0.0336,"coverage":0.9999,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.3361,"skill":0.0377,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.3401,"skill":0.1201,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.2652,"skill":0.0815,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.4836,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5112,"skill":0.0224,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.3484,"skill":0.0,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0012,"skill":0.0,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1377,"skill":0.0,"coverage":0.998,"random":0.1641,"tracks":[]},"12":{"raw":0.3048,"skill":0.0,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5083,"skill":0.0166,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.3479,"skill":0.1305,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.031,"skill":0.0,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.7008,"skill":0.5599,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.3981,"skill":0.0,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.2383,"skill":0.2285,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.2689,"skill":0.2645,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.1555,"skill":0.1392,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.7599,"pending":0.24,"latency_rows":750,"latency_path":"GLiNER2 Classifier.classify in bf16 (board fp32; bf16 is equal on median and lower on p95), fleet venv torch 2.14.0+cu130, board guard lifts"},{"engine":"gliner-decide","name":"GLiNER2.5-Decide","long_name":"GLiNER2.5-Decide","variant":null,"meta":{"kind":"full fine-tune","technique":"GLi family","technique_name":"GLiNER2","served_params":486444053,"params_estimated":false,"params_from_name":false,"trained_bytes":1945828140,"model_url":"https://huggingface.co/fastino/GLiNER2.5-Decide","code_url":"https://github.com/fastino-ai/GLiNER2","base_model":"fastino/gliner2-large-v1","base_url":"https://huggingface.co/fastino/gliner2-large-v1","weights_repo":"fastino/GLiNER2.5-Decide","served_checkpoint":null},"completed":120340,"counts":{"ok":119898},"lifted":false,"calibration":{"acc":0.434,"conf":0.506,"ece":0.0884,"brier":0.7049,"over95":0.0132,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0728,"conf":0.0668,"acc":0.0955},{"lo":0.1,"w":0.0602,"conf":0.1395,"acc":0.1625},{"lo":0.2,"w":0.0435,"conf":0.269,"acc":0.2737},{"lo":0.3,"w":0.1254,"conf":0.3513,"acc":0.3479},{"lo":0.4,"w":0.1185,"conf":0.443,"acc":0.458},{"lo":0.5,"w":0.2637,"conf":0.5461,"acc":0.5565},{"lo":0.6,"w":0.1213,"conf":0.6415,"acc":0.6031},{"lo":0.7,"w":0.0781,"conf":0.746,"acc":0.4909},{"lo":0.8,"w":0.0623,"conf":0.8508,"acc":0.4376},{"lo":0.9,"w":0.0541,"conf":0.9493,"acc":0.4055}],"areas":{"knowledge":{"ece":0.1068,"acc":0.3494,"conf":0.4562},"language":{"ece":0.1455,"acc":0.5272,"conf":0.6136},"retrieval":{"ece":0.3318,"acc":0.4856,"conf":0.5177},"tools":{"ece":0.0778,"acc":0.4236,"conf":0.4034},"arts":{"ece":0.0567,"acc":0.4176,"conf":0.4743}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":23.3,"p95":228.6,"mean":156.4},"results":{"1":{"score":0.4622,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":29.1},"2":{"score":0.382,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":4962.3},"3":{"score":0.0768,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.6563,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":42.5},"5":{"score":0.6042,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":96.5},"6":{"score":0.5935,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":142.3},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.0079,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":82.6},"11":{"score":0.2177,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.3091,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":22.2},"20":{"score":0.5816,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.4},"21":{"score":0.5065,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":21.8},"22":{"score":0.017,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1342.9},"23":{"score":0.5336,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":23.6},"24":{"score":0.3349,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":22.4},"25":{"score":0.2143,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.5412,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":22.2},"27":{"score":0.3677,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":22.5},"28":{"score":0.5067,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":22.0},"29":{"score":0.3224,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":22.8},"30":{"score":0.1766,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":22.6},"31":{"score":0.0808,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":46.1},"32":{"score":0.4521,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0073,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":30.9},"36":{"score":0.201,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.2328,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":23.3},"38":{"score":0.0,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1549.9},"39":{"score":0.6616,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.3235,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":22.5,"tracks":[{"track":"A · Arabic","score":0.0847,"headline":false},{"track":"A · English","score":0.3235,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.535,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3527,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":22.3},"42":{"score":0.4801,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":23.2},"43":{"score":0.4018,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5012,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.3},"45":{"score":0.1677,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2393,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":24.5},"50":{"score":0.4547,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":27.6},"56":{"score":0.5,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":25.2,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.161,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":22.9,"edition_added":"v0.2","jev":0.827},"58":{"score":0.3114,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":22.5,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.3341,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":27.5,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.322,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.2533,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":25.3,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5068,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":23.8,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":32.35,"balanced_skill":11.21,"breadth_skill":9.85},"frozen_scores":{"balanced_raw":26.38,"balanced_skill":5.77,"breadth_skill":5.21},"categories":[{"id":"knowledge","raw":0.2449,"skill":0.0241,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.3448,"skill":0.0748,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.4535,"skill":0.2532,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.2524,"skill":0.1478,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.3408,"skill":0.0887,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.4622,"skill":0.274,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.382,"skill":0.2863,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.5935,"skill":0.0621,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.5176,"headline":false},{"track":"RouterBench-5shot","score":0.6695,"headline":false}]},"11":{"raw":0.2177,"skill":0.0,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.5701,"skill":0.1401,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5065,"skill":0.0129,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0168,"skill":0.0091,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5321,"skill":0.0642,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3365,"skill":0.1154,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2143,"skill":0.0,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.1766,"skill":0.0027,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.254,"headline":false},{"track":"GSM8K-10choice","score":0.0993,"headline":false}]},"31":{"raw":0.0808,"skill":0.0,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.201,"skill":0.0961,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.2328,"skill":0.0378,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.3235,"skill":0.1297,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0847,"headline":false},{"track":"A · English","score":0.3235,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.535,"headline":false}]},"41":{"raw":0.3527,"skill":0.029,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4018,"skill":0.0508,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5012,"skill":0.0024,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4547,"skill":0.2085,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.161,"skill":0.0563,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.3114,"skill":0.0019,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.3341,"skill":0.1121,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.322,"skill":0.1525,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.2533,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5068,"skill":0.0136,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.4521,"skill":0.1289,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0073,"skill":0.0,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1677,"skill":0.0043,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.3091,"skill":0.0,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5067,"skill":0.0134,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.3224,"skill":0.0965,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.6616,"skill":0.5023,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.4801,"skill":0.0,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.6563,"skill":0.6519,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.6042,"skill":0.6018,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.0768,"skill":0.059,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0428,"skill":0.0428,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author AutoExtractor.classify_text with TF32 matmuls on (JEV_DECIDE_TF32=1; 23.3 / 228.6 ms vs 24.0 / 251.3 ms in fp32, kept as latency-v1-full.fp32), encoder attention eager as shipped, venv-q torch 2.14.0+cu130 (+peft)","new_in_edition":true},{"engine":"gliner-multi","name":"GLiNER 2.5 multilingual","long_name":"GLiNER 2.5 multilingual","variant":null,"meta":{"kind":"full fine-tune","technique":"GLi family","technique_name":"GLiNER2","served_params":287355159,"params_estimated":false,"params_from_name":false,"trained_bytes":1149461028,"model_url":"https://huggingface.co/fastino/gliner2.5-multi-v1","code_url":null,"base_model":null,"base_url":null,"weights_repo":"fastino/gliner2.5-multi-v1","served_checkpoint":null},"completed":120340,"counts":{"ok":120331,"unsupported":9},"lifted":true,"calibration":{"acc":0.3233,"conf":0.5771,"ece":0.2538,"brier":0.8461,"over95":0.0478,"n":72594,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0026,"conf":0.0847,"acc":0.087},{"lo":0.1,"w":0.0197,"conf":0.1554,"acc":0.026},{"lo":0.2,"w":0.0689,"conf":0.2672,"acc":0.1579},{"lo":0.3,"w":0.1963,"conf":0.3512,"acc":0.2351},{"lo":0.4,"w":0.1475,"conf":0.4483,"acc":0.236},{"lo":0.5,"w":0.1399,"conf":0.5477,"acc":0.3913},{"lo":0.6,"w":0.1128,"conf":0.6491,"acc":0.423},{"lo":0.7,"w":0.1006,"conf":0.7438,"acc":0.4335},{"lo":0.8,"w":0.0743,"conf":0.851,"acc":0.3654},{"lo":0.9,"w":0.1373,"conf":0.964,"acc":0.4188}],"areas":{"knowledge":{"ece":0.2033,"acc":0.2961,"conf":0.4993},"language":{"ece":0.2889,"acc":0.398,"conf":0.6869},"retrieval":{"ece":0.2284,"acc":0.237,"conf":0.4654},"tools":{"ece":0.4476,"acc":0.2362,"conf":0.6838},"arts":{"ece":0.2056,"acc":0.3702,"conf":0.5758}}},"gaps":{"share":0.0001,"reasons":[{"reason":"prompt text with parentheses, which the native GLiNER prompt syntax cannot carry","share":0.0001}]},"latency":{"median":14.0,"p95":92.4,"mean":46.8},"results":{"1":{"score":0.3743,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":15.3},"2":{"score":0.1316,"jev":0.6529,"metric":"nDCG@10","requests":685,"answered":684,"unsupported":1,"errors":0,"median_ms":934.8},"3":{"score":0.0157,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.0903,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":17.6},"5":{"score":0.1278,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":32.1},"6":{"score":0.5871,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":49.5},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4188,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":33.4},"11":{"score":0.3021,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.3079,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":13.2},"20":{"score":0.4926,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":13.1},"21":{"score":0.5209,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":12.8},"22":{"score":0.034,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":291.1},"23":{"score":0.4958,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.4},"24":{"score":0.2811,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":13.1},"25":{"score":0.2347,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.3527,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2373,"unsupported":3,"errors":0,"median_ms":12.9},"27":{"score":0.2568,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":13.0},"28":{"score":0.5114,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":12.7},"29":{"score":0.3007,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":13.6},"30":{"score":0.1945,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":13.5},"31":{"score":0.0738,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":15.4},"32":{"score":0.3404,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":19.7},"36":{"score":0.1185,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.1498,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.2},"38":{"score":0.0301,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":501.2},"39":{"score":0.6852,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2376,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":12.9,"tracks":[{"track":"A · Arabic","score":0.221,"headline":false},{"track":"A · English","score":0.2376,"headline":true},{"track":"C · Arabic pairs","score":0.33,"headline":false},{"track":"C · English pairs","score":0.375,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.2642,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":13.3},"42":{"score":0.4129,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":14.0},"43":{"score":0.4035,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.4926,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":13.1},"45":{"score":0.132,"jev":0.204,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":null},"48":{"score":0.3341,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":14.6},"50":{"score":0.3598,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":16.0},"56":{"score":0.5025,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":15.4,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.1199,"metric":"accuracy","requests":12032,"answered":12031,"unsupported":1,"errors":0,"median_ms":13.6,"edition_added":"v0.2","jev":0.827},"58":{"score":0.3138,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":13.5,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.3478,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":15.8,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.233,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.5223,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":15.5,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5505,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":14.9,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":26.2,"balanced_skill":4.26,"breadth_skill":4.09},"frozen_scores":{"balanced_raw":23.19,"balanced_skill":1.84,"breadth_skill":1.75},"categories":[{"id":"knowledge","raw":0.2282,"skill":0.009,"coverage":0.9998,"pending":0,"n":10},{"id":"language","raw":0.3583,"skill":0.0636,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.2434,"skill":0.0531,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.1754,"skill":0.0587,"coverage":0.9997,"pending":0,"n":5},{"id":"arts","raw":0.2955,"skill":0.025,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.3743,"skill":0.1553,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.1314,"skill":0.0,"coverage":0.9985,"random":0.1341,"tracks":[]},"6":{"raw":0.5872,"skill":0.0316,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.5411,"headline":false},{"track":"RouterBench-5shot","score":0.6332,"headline":false}]},"11":{"raw":0.3021,"skill":0.0,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.4948,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5209,"skill":0.0419,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0337,"skill":0.0262,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.4851,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.2862,"skill":0.0483,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2347,"skill":0.0,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.1945,"skill":0.025,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.2805,"headline":false},{"track":"GSM8K-10choice","score":0.1084,"headline":false}]},"31":{"raw":0.0738,"skill":0.0,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.1185,"skill":0.0027,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.1498,"skill":0.0,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.2376,"skill":0.0192,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.221,"headline":false},{"track":"A · English","score":0.2376,"headline":true},{"track":"C · Arabic pairs","score":0.33,"headline":false},{"track":"C · English pairs","score":0.375,"headline":false}]},"41":{"raw":0.2642,"skill":0.0,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4035,"skill":0.0536,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.4926,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3598,"skill":0.0708,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5025,"skill":0.005,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.1198,"skill":0.01,"coverage":0.9999,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.3138,"skill":0.0054,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.3478,"skill":0.1304,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.233,"skill":0.0413,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.5223,"skill":0.0095,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5505,"skill":0.101,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.3404,"skill":0.0,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1317,"skill":0.0,"coverage":0.998,"random":0.1641,"tracks":[]},"12":{"raw":0.3079,"skill":0.0,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5114,"skill":0.0228,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.3007,"skill":0.0676,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0301,"skill":0.0,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.6852,"skill":0.537,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.4129,"skill":0.0,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.0903,"skill":0.0786,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.1278,"skill":0.1225,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.0157,"skill":0.0,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.7599,"pending":0.24,"latency_rows":750,"latency_path":"GLiNER2 Classifier.classify in bf16 (board fp32; bf16 is equal on median and lower on p95), fleet venv torch 2.14.0+cu130, board guard lifts"},{"engine":"gliner-small","name":"GLiNER 2.5 small","long_name":"GLiNER 2.5 small","variant":null,"meta":{"kind":"full fine-tune","technique":"GLi family","technique_name":"GLiNER2","served_params":73881879,"params_estimated":false,"params_from_name":false,"trained_bytes":295567700,"model_url":"https://huggingface.co/fastino/gliner2.5-small-v1","code_url":null,"base_model":null,"base_url":null,"weights_repo":"fastino/gliner2.5-small-v1","served_checkpoint":null},"completed":120340,"counts":{"ok":120331,"unsupported":9},"lifted":true,"calibration":{"acc":0.3319,"conf":0.4921,"ece":0.1608,"brier":0.7667,"over95":0.024,"n":72594,"benchmarks":32,"rel":[{"lo":0.0,"w":0.1181,"conf":0.0403,"acc":0.0428},{"lo":0.1,"w":0.0513,"conf":0.1456,"acc":0.0889},{"lo":0.2,"w":0.0621,"conf":0.2701,"acc":0.2529},{"lo":0.3,"w":0.1322,"conf":0.3482,"acc":0.2965},{"lo":0.4,"w":0.1097,"conf":0.4512,"acc":0.3645},{"lo":0.5,"w":0.1733,"conf":0.5477,"acc":0.4706},{"lo":0.6,"w":0.1355,"conf":0.6408,"acc":0.3792},{"lo":0.7,"w":0.0768,"conf":0.7485,"acc":0.4508},{"lo":0.8,"w":0.0614,"conf":0.8481,"acc":0.3855},{"lo":0.9,"w":0.0795,"conf":0.9583,"acc":0.4541}],"areas":{"knowledge":{"ece":0.1318,"acc":0.3144,"conf":0.4461},"language":{"ece":0.3329,"acc":0.3208,"conf":0.6537},"retrieval":{"ece":0.0742,"acc":0.3214,"conf":0.3478},"tools":{"ece":0.0761,"acc":0.3317,"conf":0.4049},"arts":{"ece":0.0873,"acc":0.3987,"conf":0.4703}}},"gaps":{"share":0.0001,"reasons":[{"reason":"prompt text with parentheses, which the native GLiNER prompt syntax cannot carry","share":0.0001}]},"latency":{"median":13.6,"p95":35.2,"mean":31.2},"results":{"1":{"score":0.4067,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":15.3},"2":{"score":0.1624,"jev":0.6529,"metric":"nDCG@10","requests":685,"answered":684,"unsupported":1,"errors":0,"median_ms":736.5},"3":{"score":0.0197,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.0854,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":17.5},"5":{"score":0.136,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":23.9},"6":{"score":0.5725,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":21.7},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.479,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":15.7},"11":{"score":0.211,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.2537,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":12.7},"20":{"score":0.4814,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":12.8},"21":{"score":0.5023,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":12.6},"22":{"score":0.0465,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":200.4},"23":{"score":0.5182,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.2},"24":{"score":0.3066,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":12.8},"25":{"score":0.2653,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.4282,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2373,"unsupported":3,"errors":0,"median_ms":12.7},"27":{"score":0.2807,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":12.7},"28":{"score":0.4988,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":12.7},"29":{"score":0.3169,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":13.2},"30":{"score":0.2346,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":13.1},"31":{"score":0.0808,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":15.5},"32":{"score":0.3777,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0012,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":17.9},"36":{"score":0.0979,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.2082,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":13.8},"38":{"score":0.031,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":237.8},"39":{"score":0.4662,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2373,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":12.7,"tracks":[{"track":"A · Arabic","score":0.2475,"headline":false},{"track":"A · English","score":0.2373,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.49,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3708,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":13.0},"42":{"score":0.4628,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":14.0},"43":{"score":0.4,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.4922,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":12.8},"45":{"score":0.152,"jev":0.204,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":null},"48":{"score":0.5084,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":14.4},"50":{"score":0.3311,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":15.4},"56":{"score":0.5,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":15.3,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.1233,"metric":"accuracy","requests":12032,"answered":12031,"unsupported":1,"errors":0,"median_ms":13.4,"edition_added":"v0.2","jev":0.827},"58":{"score":0.3178,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":13.0,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.3283,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":15.2,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.2841,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.4099,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":14.9,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.516,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":14.4,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":25.99,"balanced_skill":3.82,"breadth_skill":3.72},"frozen_scores":{"balanced_raw":24.2,"balanced_skill":2.54,"breadth_skill":2.41},"categories":[{"id":"knowledge","raw":0.2422,"skill":0.0175,"coverage":0.9998,"pending":0,"n":10},{"id":"language","raw":0.3276,"skill":0.0375,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.2435,"skill":0.043,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.1856,"skill":0.0698,"coverage":0.9997,"pending":0,"n":5},{"id":"arts","raw":0.2991,"skill":0.0266,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.4067,"skill":0.1991,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.1622,"skill":0.0324,"coverage":0.9985,"random":0.1341,"tracks":[]},"6":{"raw":0.5725,"skill":0.0168,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.5098,"headline":false},{"track":"RouterBench-5shot","score":0.6352,"headline":false}]},"11":{"raw":0.211,"skill":0.0,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.4838,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5023,"skill":0.0046,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0448,"skill":0.0373,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5076,"skill":0.0153,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3093,"skill":0.0791,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2653,"skill":0.0204,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.2346,"skill":0.0756,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.3343,"headline":false},{"track":"GSM8K-10choice","score":0.135,"headline":false}]},"31":{"raw":0.0808,"skill":0.0,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.0979,"skill":0.0,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.2082,"skill":0.0068,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.2373,"skill":0.0188,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2475,"headline":false},{"track":"A · English","score":0.2373,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.49,"headline":false}]},"41":{"raw":0.3708,"skill":0.0563,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4,"skill":0.048,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.4922,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3311,"skill":0.0293,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.1233,"skill":0.0139,"coverage":0.9999,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.3178,"skill":0.0112,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.3283,"skill":0.1044,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.2841,"skill":0.1051,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.4099,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.516,"skill":0.032,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.3777,"skill":0.0107,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0012,"skill":0.0,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1517,"skill":0.0,"coverage":0.998,"random":0.1641,"tracks":[]},"12":{"raw":0.2537,"skill":0.0,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.4988,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.3169,"skill":0.0892,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.031,"skill":0.0,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.4662,"skill":0.2149,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.4628,"skill":0.0,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.0854,"skill":0.0736,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.136,"skill":0.1308,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.0197,"skill":0.0008,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.7599,"pending":0.24,"latency_rows":750,"latency_path":"GLiNER2 Classifier.classify in bf16 (board fp32; bf16 is equal on median and lower on p95), fleet venv torch 2.14.0+cu130, board guard lifts"},{"engine":"harsha","name":"Qwen-2.5-1B-RLCD","long_name":"Qwen-2.5-1B-RLCD","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":1543714304,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://huggingface.co/harshatheg/Qwen-2.5-1B-RLCD","base_model":"Qwen/Qwen2.5-1.5B","base_url":"https://huggingface.co/Qwen/Qwen2.5-1.5B","weights_repo":null,"served_checkpoint":null},"completed":120340,"counts":{"ok":119960,"unsupported":380},"lifted":false,"calibration":{"acc":0.3648,"conf":0.5671,"ece":0.2052,"brier":0.7934,"over95":0.0352,"n":72594,"benchmarks":32,"rel":[{"lo":0.0,"w":0.1428,"conf":0.0392,"acc":0.0474},{"lo":0.1,"w":0.01,"conf":0.1432,"acc":0.1705},{"lo":0.2,"w":0.019,"conf":0.2576,"acc":0.121},{"lo":0.3,"w":0.0613,"conf":0.3546,"acc":0.2552},{"lo":0.4,"w":0.1033,"conf":0.4547,"acc":0.3506},{"lo":0.5,"w":0.1708,"conf":0.5484,"acc":0.4176},{"lo":0.6,"w":0.1227,"conf":0.6483,"acc":0.4143},{"lo":0.7,"w":0.1501,"conf":0.7495,"acc":0.475},{"lo":0.8,"w":0.094,"conf":0.8516,"acc":0.4813},{"lo":0.9,"w":0.1261,"conf":0.9578,"acc":0.5039}],"areas":{"knowledge":{"ece":0.199,"acc":0.3585,"conf":0.5571},"language":{"ece":0.2699,"acc":0.4209,"conf":0.6906},"retrieval":{"ece":0.2027,"acc":0.1929,"conf":0.3787},"tools":{"ece":0.2339,"acc":0.347,"conf":0.5761},"arts":{"ece":0.1175,"acc":0.4261,"conf":0.5121}}},"gaps":{"share":0.0032,"reasons":[{"reason":"prompt longer than its context window (no truncation allowed)","share":0.0032}]},"latency":{"median":43.8,"p95":75.2,"mean":53.6},"results":{"1":{"score":0.474,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":45.8},"2":{"score":0.0852,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":257.4},"3":{"score":0.0433,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.0259,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":47.9},"5":{"score":0.0116,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":56.4},"6":{"score":0.5886,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":65.9},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4644,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":47.6},"11":{"score":0.2668,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.2819,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":42.9},"20":{"score":0.6808,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":43.2},"21":{"score":0.5068,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":42.0},"22":{"score":0.029,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":111.6},"23":{"score":0.5024,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":44.0},"24":{"score":0.3688,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":42.7},"25":{"score":0.2806,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.5648,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":42.5},"27":{"score":0.4369,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":42.2},"28":{"score":0.5043,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":42.0},"29":{"score":0.4255,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":43.5},"30":{"score":0.1748,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":43.2},"31":{"score":0.0882,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":45.6},"32":{"score":0.3777,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0236,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":47.6},"36":{"score":0.1433,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.0892,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":43.9},"38":{"score":0.0458,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":295.6},"39":{"score":0.4332,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2431,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":42.2,"tracks":[{"track":"A · Arabic","score":0.2768,"headline":false},{"track":"A · English","score":0.2431,"headline":true},{"track":"C · Arabic pairs","score":0.495,"headline":false},{"track":"C · English pairs","score":0.475,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3142,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":43.3},"42":{"score":0.4373,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":44.4},"43":{"score":0.3579,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.503,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":43.1},"45":{"score":0.1717,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2466,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":45.6},"50":{"score":0.3848,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":46.7},"56":{"score":0.4905,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":45.6,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.1154,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":43.2,"edition_added":"v0.2","jev":0.827},"58":{"score":0.3374,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":42.7,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.2204,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":45.1,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.1932,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.0063,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":45.8,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5012,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":44.1,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":24.34,"balanced_skill":3.78,"breadth_skill":3.56},"frozen_scores":{"balanced_raw":24.91,"balanced_skill":4.51,"breadth_skill":4.08},"categories":[{"id":"knowledge","raw":0.2418,"skill":0.014,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.302,"skill":0.0471,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.1966,"skill":0.0094,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.1715,"skill":0.07,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.3214,"skill":0.0731,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.474,"skill":0.29,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.0852,"skill":0.0,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.5886,"skill":0.0372,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.5345,"headline":false},{"track":"RouterBench-5shot","score":0.6427,"headline":false}]},"11":{"raw":0.2668,"skill":0.0,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.6735,"skill":0.347,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5068,"skill":0.0137,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0298,"skill":0.0222,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.51,"skill":0.02,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3832,"skill":0.1776,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2806,"skill":0.0408,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.1748,"skill":0.003,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.2441,"headline":false},{"track":"GSM8K-10choice","score":0.1054,"headline":false}]},"31":{"raw":0.0882,"skill":0.0069,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.1433,"skill":0.0308,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.0892,"skill":0.0,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.2431,"skill":0.0262,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2768,"headline":false},{"track":"A · English","score":0.2431,"headline":true},{"track":"C · Arabic pairs","score":0.495,"headline":false},{"track":"C · English pairs","score":0.475,"headline":false}]},"41":{"raw":0.3142,"skill":0.0,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3579,"skill":0.0,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.503,"skill":0.006,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3848,"skill":0.1072,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.4905,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.1154,"skill":0.0051,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.3374,"skill":0.0396,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.2204,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.1932,"skill":0.0,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.0063,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5012,"skill":0.0024,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.3777,"skill":0.0107,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0236,"skill":0.0106,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1717,"skill":0.0091,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.2819,"skill":0.0,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5043,"skill":0.0086,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.4255,"skill":0.234,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0458,"skill":0.0153,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.4332,"skill":0.1663,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.4373,"skill":0.0,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.0259,"skill":0.0134,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.0116,"skill":0.0056,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.0433,"skill":0.0249,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0136,"skill":0.0136,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author engine_torch (bf16, SDPA, shared prefix + batched candidate suffixes), fleet venv torch 2.14.0+cu130; same as board"},{"engine":"hopper-g-1.2","name":"Hopper (G) 1.2","long_name":"Hopper (G) 1.2 Qwen3.5-4B LoRA","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":129927008,"model_url":"https://huggingface.co/HopitAI/hopper-g","code_url":"https://github.com/hopit-ai/hopper","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"HopitAI/hopper-g","served_checkpoint":null},"completed":120340,"counts":{"ok":119898},"lifted":false,"calibration":{"acc":0.651,"conf":0.7437,"ece":0.0927,"brier":0.4672,"over95":0.0021,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0004,"conf":0.0937,"acc":0.0},{"lo":0.1,"w":0.0086,"conf":0.1646,"acc":0.0574},{"lo":0.2,"w":0.0249,"conf":0.2509,"acc":0.1464},{"lo":0.3,"w":0.0389,"conf":0.3478,"acc":0.2482},{"lo":0.4,"w":0.0634,"conf":0.4538,"acc":0.2713},{"lo":0.5,"w":0.1087,"conf":0.5493,"acc":0.4367},{"lo":0.6,"w":0.1139,"conf":0.6509,"acc":0.5037},{"lo":0.7,"w":0.1206,"conf":0.7519,"acc":0.6122},{"lo":0.8,"w":0.216,"conf":0.8544,"acc":0.7424},{"lo":0.9,"w":0.3046,"conf":0.9343,"acc":0.9225}],"areas":{"knowledge":{"ece":0.1264,"acc":0.553,"conf":0.6793},"language":{"ece":0.0313,"acc":0.7484,"conf":0.778},"retrieval":{"ece":0.1105,"acc":0.6944,"conf":0.8042},"tools":{"ece":0.039,"acc":0.8847,"conf":0.872},"arts":{"ece":0.1815,"acc":0.5166,"conf":0.6981}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":23.3,"p95":238.2,"mean":105.1},"results":{"1":{"score":0.9209,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":94.5},"2":{"score":0.6195,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":798.6},"3":{"score":0.7559,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7255,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":114.1},"5":{"score":0.7957,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":209.1},"6":{"score":0.7986,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":186.4},"9":{"score":0.2273,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6438,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":65.4},"11":{"score":0.7073,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5995,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":14.5},"20":{"score":0.8322,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.5},"21":{"score":0.6058,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":13.3},"22":{"score":0.068,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":741.7},"23":{"score":0.6064,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":36.1},"24":{"score":0.7389,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":15.3},"25":{"score":0.3776,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9739,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":15.2},"27":{"score":0.9275,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":14.6},"28":{"score":0.6914,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":13.3},"29":{"score":0.8451,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":21.4},"30":{"score":0.4795,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":18.7},"31":{"score":0.1392,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":100.6},"32":{"score":0.5505,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2952,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":202.5},"36":{"score":0.4244,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3951,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.3},"38":{"score":0.1645,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":889.5},"39":{"score":0.8319,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4371,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":13.4,"tracks":[{"track":"A · Arabic","score":0.3065,"headline":false},{"track":"A · English","score":0.4371,"headline":true},{"track":"C · Arabic pairs","score":0.565,"headline":false},{"track":"C · English pairs","score":0.885,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4432,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":18.7},"42":{"score":0.7857,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":33.2},"43":{"score":0.4719,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6302,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.4},"45":{"score":0.0918,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2243,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":36.7},"50":{"score":0.4272,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":42.8},"56":{"score":0.6135,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":185.1,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.4894,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":21.3,"edition_added":"v0.2","jev":0.827},"58":{"score":0.6414,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":15.2,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.6117,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":43.9,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.5758,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.6957,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":36.6,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.6455,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":32.2,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":55.23,"balanced_skill":40.77,"breadth_skill":39.16},"frozen_scores":{"balanced_raw":40.9,"balanced_skill":26.41,"breadth_skill":22.73},"categories":[{"id":"knowledge","raw":0.4154,"skill":0.2517,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.6253,"skill":0.4483,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.6044,"skill":0.4542,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.6427,"skill":0.6006,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4473,"skill":0.2602,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9209,"skill":0.8932,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6195,"skill":0.5606,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7986,"skill":0.5252,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7848,"headline":false},{"track":"RouterBench-5shot","score":0.8124,"headline":false}]},"11":{"raw":0.7073,"skill":0.5768,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.8352,"skill":0.6703,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6058,"skill":0.2116,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0697,"skill":0.0624,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5836,"skill":0.1673,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7487,"skill":0.6649,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3776,"skill":0.1701,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.4795,"skill":0.3707,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.5406,"headline":false},{"track":"GSM8K-10choice","score":0.4185,"headline":false}]},"31":{"raw":0.1392,"skill":0.0625,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4244,"skill":0.3488,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.3951,"skill":0.2412,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.4371,"skill":0.2758,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3065,"headline":false},{"track":"A · English","score":0.4371,"headline":true},{"track":"C · Arabic pairs","score":0.565,"headline":false},{"track":"C · English pairs","score":0.885,"headline":false}]},"41":{"raw":0.4432,"skill":0.1649,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4719,"skill":0.1622,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6302,"skill":0.2604,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4272,"skill":0.1687,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.6135,"skill":0.227,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.4894,"skill":0.4257,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6414,"skill":0.4802,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.6117,"skill":0.4823,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.5758,"skill":0.4697,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.6957,"skill":0.3691,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.6455,"skill":0.291,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5505,"skill":0.2854,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2952,"skill":0.2858,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.0918,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.5995,"skill":0.4001,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.6914,"skill":0.3828,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.8451,"skill":0.7935,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.1645,"skill":0.1378,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8319,"skill":0.7528,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7857,"skill":0.5831,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.7255,"skill":0.722,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.7957,"skill":0.7945,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.7559,"skill":0.7512,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.2273,"skill":0.2273,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.1028,"skill":0.1028,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"hopper-decisions 1.1.1 at tag g-1.2.0 with the HopitAI/hopper-g adapter (tournament shortlist for choice questions over 26 options, package default), Decider(cuda_graphs=True) package opt-in CUDA graphs per length bucket, eager outside captured buckets; author pins torch 2.8.0+cu128 / transformers 5.17.0 / fla 0.5.2 / causal-conv1d 1.7.0; eager variant kept as latency-v1-full.eager","new_in_edition":true},{"engine":"jeff-uncapped","name":"jeff","long_name":"jeff GLiFormer-large","variant":null,"meta":{"kind":"inference technique","technique":"GLi family","technique_name":"GLiFormer","served_params":575683963,"params_estimated":true,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/logan-markewich/jeff","base_model":"knowledgator/gliformer-large-v1","base_url":"https://huggingface.co/knowledgator/gliformer-large-v1","weights_repo":null,"served_checkpoint":"knowledgator/gliformer-large-v1"},"completed":120340,"counts":{"ok":120340},"lifted":true,"calibration":{"acc":0.3824,"conf":0.4197,"ece":0.0969,"brier":0.6823,"over95":0,"n":72594,"benchmarks":32,"rel":[{"lo":0.0,"w":0.1485,"conf":0.037,"acc":0.2279},{"lo":0.1,"w":0.0306,"conf":0.146,"acc":0.1938},{"lo":0.2,"w":0.0665,"conf":0.2737,"acc":0.2396},{"lo":0.3,"w":0.2018,"conf":0.3492,"acc":0.3442},{"lo":0.4,"w":0.1022,"conf":0.4394,"acc":0.4299},{"lo":0.5,"w":0.2356,"conf":0.5454,"acc":0.4335},{"lo":0.6,"w":0.1335,"conf":0.646,"acc":0.5012},{"lo":0.7,"w":0.0683,"conf":0.7403,"acc":0.5444},{"lo":0.8,"w":0.0129,"conf":0.8286,"acc":0.549},{"lo":0.9,"w":0.0002,"conf":0.9121,"acc":0.0968}],"areas":{"knowledge":{"ece":0.0713,"acc":0.3278,"conf":0.3991},"language":{"ece":0.1031,"acc":0.4133,"conf":0.5119},"retrieval":{"ece":0.3082,"acc":0.553,"conf":0.2595},"tools":{"ece":0.1054,"acc":0.3014,"conf":0.3827},"arts":{"ece":0.096,"acc":0.3592,"conf":0.4491}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":21.8,"p95":67.6,"mean":33.1},"results":{"1":{"score":0.2149,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":35.0},"2":{"score":0.1763,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":243.5},"3":{"score":0.0315,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.6099,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":21.4},"5":{"score":0.443,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":23.0},"6":{"score":0.5384,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":46.6},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4479,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":49.7},"11":{"score":0.2126,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.2934,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":19.2},"20":{"score":0.4814,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":19.2},"21":{"score":0.4665,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":18.8},"22":{"score":0.024,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":112.3},"23":{"score":0.5044,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":26.4},"24":{"score":0.316,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":19.2},"25":{"score":0.2449,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.5059,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":18.9},"27":{"score":0.3166,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":18.9},"28":{"score":0.5036,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":18.6},"29":{"score":0.3455,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":19.6},"30":{"score":0.2002,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":20.8},"31":{"score":0.086,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.6},"32":{"score":0.4787,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0158,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":29.0},"36":{"score":0.0808,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.2128,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":23.2},"38":{"score":0.0316,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":102.1},"39":{"score":0.7487,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2832,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":19.4,"tracks":[{"track":"A · Arabic","score":0.0679,"headline":false},{"track":"A · English","score":0.2832,"headline":true},{"track":"C · Arabic pairs","score":0.52,"headline":false},{"track":"C · English pairs","score":0.37,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4047,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":19.2},"42":{"score":0.5504,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":28.2},"43":{"score":0.3895,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.4956,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":19.3},"45":{"score":0.1657,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2906,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":32.8},"50":{"score":0.3252,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":29.3},"56":{"score":0.5025,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":25.8,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.1423,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":20.1,"edition_added":"v0.2","jev":0.827},"58":{"score":0.335,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":21.1,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.2333,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":33.4,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.1875,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.171,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":34.1,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5677,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":27.3,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":28.91,"balanced_skill":8.04,"breadth_skill":6.72},"frozen_scores":{"balanced_raw":22.75,"balanced_skill":1.64,"breadth_skill":1.58},"categories":[{"id":"knowledge","raw":0.2529,"skill":0.0305,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.356,"skill":0.104,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.4005,"skill":0.2131,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.1306,"skill":0.0119,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.2766,"skill":0.0077,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.2149,"skill":0.0,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.1763,"skill":0.0488,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.5385,"skill":0.0,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.4941,"headline":false},{"track":"RouterBench-5shot","score":0.5828,"headline":false}]},"11":{"raw":0.2126,"skill":0.0,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.4807,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.4665,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0204,"skill":0.0127,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.511,"skill":0.0221,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.316,"skill":0.088,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2449,"skill":0.0,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.2002,"skill":0.0318,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.2851,"headline":false},{"track":"GSM8K-10choice","score":0.1152,"headline":false}]},"31":{"raw":0.086,"skill":0.0045,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.0808,"skill":0.0,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.2128,"skill":0.0127,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.2832,"skill":0.0778,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0679,"headline":false},{"track":"A · English","score":0.2832,"headline":true},{"track":"C · Arabic pairs","score":0.52,"headline":false},{"track":"C · English pairs","score":0.37,"headline":false}]},"41":{"raw":0.4047,"skill":0.107,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3895,"skill":0.0313,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.4956,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3252,"skill":0.0206,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5025,"skill":0.005,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.1423,"skill":0.0353,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.335,"skill":0.0361,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.2333,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.1875,"skill":0.0,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.171,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5677,"skill":0.1354,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.4787,"skill":0.1712,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0158,"skill":0.0027,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1657,"skill":0.0019,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.2934,"skill":0.0,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5036,"skill":0.0072,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.3455,"skill":0.1273,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0316,"skill":0.0006,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.7487,"skill":0.6304,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.5504,"skill":0.1253,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.6099,"skill":0.6049,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.443,"skill":0.4396,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.0315,"skill":0.0128,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author jeff encoder with flash attention, long-context lift as board (compile off in the package); venv-jeff"},{"engine":"jev-omni","name":"Jev-Omni","long_name":"Jev-Omni","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":11959730224,"params_estimated":false,"params_from_name":false,"trained_bytes":23923515487,"model_url":"https://huggingface.co/akhilaaa3/Jev-Omni","code_url":null,"base_model":"google/gemma-4-12B","base_url":"https://huggingface.co/google/gemma-4-12B","weights_repo":"akhilaaa3/Jev-Omni","served_checkpoint":null},"completed":120340,"counts":{"ok":119888},"lifted":false,"calibration":{"acc":0.6137,"conf":0.7748,"ece":0.161,"brier":0.5344,"over95":0.0552,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0172,"conf":0.0763,"acc":0.0247},{"lo":0.1,"w":0.0431,"conf":0.1469,"acc":0.0401},{"lo":0.2,"w":0.0309,"conf":0.2471,"acc":0.1333},{"lo":0.3,"w":0.0367,"conf":0.353,"acc":0.2692},{"lo":0.4,"w":0.0414,"conf":0.4507,"acc":0.2998},{"lo":0.5,"w":0.0749,"conf":0.5541,"acc":0.3944},{"lo":0.6,"w":0.0764,"conf":0.649,"acc":0.4738},{"lo":0.7,"w":0.0725,"conf":0.7527,"acc":0.5175},{"lo":0.8,"w":0.0979,"conf":0.8536,"acc":0.577},{"lo":0.9,"w":0.5091,"conf":0.9796,"acc":0.8357}],"areas":{"knowledge":{"ece":0.1741,"acc":0.5739,"conf":0.748},"language":{"ece":0.1559,"acc":0.7534,"conf":0.9089},"retrieval":{"ece":0.2051,"acc":0.4204,"conf":0.6254},"tools":{"ece":0.0448,"acc":0.81,"conf":0.7923},"arts":{"ece":0.2145,"acc":0.4867,"conf":0.7012}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":54.9,"p95":627.9,"mean":189.4},"results":{"1":{"score":0.9675,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":149.5},"2":{"score":0.6443,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1812.2},"3":{"score":0.5236,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.3099,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":131.7},"5":{"score":0.1689,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":193.0},"6":{"score":0.7757,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":475.6},"9":{"score":0.4205,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5043,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":116.5},"11":{"score":0.6984,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6467,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":51.9},"20":{"score":0.7668,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":52.1},"21":{"score":0.6149,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":51.8},"22":{"score":0.042,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":780.7},"23":{"score":0.605,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":62.2},"24":{"score":0.7753,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":52.1},"25":{"score":0.3673,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9798,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":51.8},"27":{"score":0.9497,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":51.7},"28":{"score":0.7206,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":51.8},"29":{"score":0.8105,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":52.3},"30":{"score":0.4022,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":52.0},"31":{"score":0.1282,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":104.6},"32":{"score":0.5572,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2667,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":530.6},"36":{"score":0.4651,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.535,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":53.7},"38":{"score":0.16,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":3310.9},"39":{"score":0.9081,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4107,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":52.3,"tracks":[{"track":"A · Arabic","score":0.4766,"headline":false},{"track":"A · English","score":0.4107,"headline":true},{"track":"C · Arabic pairs","score":0.875,"headline":false},{"track":"C · English pairs","score":0.965,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.5462,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":51.9},"42":{"score":0.8232,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":71.7},"43":{"score":0.507,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6676,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":52.3},"45":{"score":0.0998,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2018,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":97.4},"50":{"score":0.4153,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":104.0},"56":{"score":0.7995,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":466.9,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.4894,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":52.3,"edition_added":"v0.2","jev":0.827},"58":{"score":0.6657,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":52.0,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.6303,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":84.1,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.5985,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.7998,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":92.0,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.6585,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":62.3,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":53.5,"balanced_skill":40.53,"breadth_skill":38.76},"frozen_scores":{"balanced_raw":42.43,"balanced_skill":28.69,"breadth_skill":24.07},"categories":[{"id":"knowledge","raw":0.4144,"skill":0.2549,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.6566,"skill":0.5052,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.4736,"skill":0.3583,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.6453,"skill":0.6077,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4537,"skill":0.2607,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9675,"skill":0.9562,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6443,"skill":0.5893,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7757,"skill":0.4699,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7657,"headline":false},{"track":"RouterBench-5shot","score":0.7858,"headline":false}]},"11":{"raw":0.6984,"skill":0.5639,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.7663,"skill":0.5326,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6149,"skill":0.2298,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0395,"skill":0.032,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.6008,"skill":0.2016,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7849,"skill":0.7132,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3673,"skill":0.1565,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.4022,"skill":0.2788,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.4875,"headline":false},{"track":"GSM8K-10choice","score":0.3169,"headline":false}]},"31":{"raw":0.1282,"skill":0.0505,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4651,"skill":0.3949,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.535,"skill":0.4167,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.4107,"skill":0.2419,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4766,"headline":false},{"track":"A · English","score":0.4107,"headline":true},{"track":"C · Arabic pairs","score":0.875,"headline":false},{"track":"C · English pairs","score":0.965,"headline":false}]},"41":{"raw":0.5462,"skill":0.3192,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.507,"skill":0.2178,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6676,"skill":0.3352,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4153,"skill":0.1514,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.7995,"skill":0.599,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.4894,"skill":0.4257,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6657,"skill":0.5154,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.6303,"skill":0.5071,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.5985,"skill":0.4981,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.7998,"skill":0.5849,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.6585,"skill":0.317,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5572,"skill":0.296,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2667,"skill":0.257,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.0998,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.6467,"skill":0.4708,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.7206,"skill":0.4412,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.8105,"skill":0.7473,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.16,"skill":0.1331,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.9081,"skill":0.8648,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.8232,"skill":0.656,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.3099,"skill":0.301,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.1689,"skill":0.1639,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.5236,"skill":0.5144,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.4205,"skill":0.4205,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.1928,"skill":0.1928,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author JevOmni.predict per question, single process (no shared-card lock); torch 2.14.0+cu130 / transformers 5.17.0","new_in_edition":true},{"engine":"jevfire-uncapped","name":"Jevfire","long_name":"Jevfire Qwen3.8-27B","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/kikoncuo/jevfire","base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":"Qwen/Qwen3.8-27B-FP8"},"completed":120340,"counts":{"ok":120340},"lifted":true,"calibration":{"acc":0.6894,"conf":0.7415,"ece":0.0521,"brier":0.4139,"over95":0.0152,"n":72594,"benchmarks":32,"rel":[{"lo":0.0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0169,"conf":0.171,"acc":0.0521},{"lo":0.2,"w":0.0335,"conf":0.2552,"acc":0.1665},{"lo":0.3,"w":0.045,"conf":0.3573,"acc":0.2594},{"lo":0.4,"w":0.0687,"conf":0.4518,"acc":0.4308},{"lo":0.5,"w":0.1472,"conf":0.5496,"acc":0.5063},{"lo":0.6,"w":0.1004,"conf":0.6485,"acc":0.5991},{"lo":0.7,"w":0.1045,"conf":0.7494,"acc":0.7231},{"lo":0.8,"w":0.105,"conf":0.8542,"acc":0.7528},{"lo":0.9,"w":0.3788,"conf":0.9739,"acc":0.9302}],"areas":{"knowledge":{"ece":0.0643,"acc":0.6084,"conf":0.6726},"language":{"ece":0.031,"acc":0.7762,"conf":0.8035},"retrieval":{"ece":0.0794,"acc":0.7266,"conf":0.804},"tools":{"ece":0.0466,"acc":0.9297,"conf":0.9266},"arts":{"ece":0.0845,"acc":0.5374,"conf":0.6201}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":89.6,"p95":447.5,"mean":178.1},"results":{"1":{"score":0.9557,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":154.7},"2":{"score":0.649,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1575.7},"3":{"score":0.8406,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7517,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":117.7},"5":{"score":0.8128,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":292.3},"6":{"score":0.7995,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":305.6},"9":{"score":0.4773,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6497,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":134.1},"11":{"score":0.7547,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6355,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":88.3},"20":{"score":0.9492,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":88.0},"21":{"score":0.5784,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":87.7},"22":{"score":0.132,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":762.1},"23":{"score":0.6046,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":90.0},"24":{"score":0.7971,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":88.2},"25":{"score":0.4694,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9848,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":87.5},"27":{"score":0.9599,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":87.8},"28":{"score":0.7451,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":87.8},"29":{"score":0.9046,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":88.6},"30":{"score":0.5489,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":88.3},"31":{"score":0.1732,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":116.9},"32":{"score":0.5944,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2752,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":490.7},"36":{"score":0.4922,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5351,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":89.1},"38":{"score":0.2238,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1455.5},"39":{"score":0.8032,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5324,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":88.0,"tracks":[{"track":"A · Arabic","score":0.3969,"headline":false},{"track":"A · English","score":0.5324,"headline":true},{"track":"C · Arabic pairs","score":0.72,"headline":false},{"track":"C · English pairs","score":0.935,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.6081,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":88.3},"42":{"score":0.7522,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":90.5},"43":{"score":0.5368,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6178,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":88.0},"45":{"score":0.1198,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2129,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":99.4},"50":{"score":0.395,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":108.0},"56":{"score":0.7435,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":361.6,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.5727,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":88.5,"edition_added":"v0.2","jev":0.827},"58":{"score":0.6797,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":88.4,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.7755,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":97.2,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.6875,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.8026,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":99.1,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.7117,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":89.9,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":61.63,"balanced_skill":49.37,"breadth_skill":47.38},"frozen_scores":{"balanced_raw":45.03,"balanced_skill":32.24,"breadth_skill":27.47},"categories":[{"id":"knowledge","raw":0.4589,"skill":0.3048,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.6811,"skill":0.5334,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.6755,"skill":0.5622,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.7514,"skill":0.7233,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4907,"skill":0.3221,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9557,"skill":0.9402,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.649,"skill":0.5947,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7996,"skill":0.5273,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7866,"headline":false},{"track":"RouterBench-5shot","score":0.8125,"headline":false}]},"11":{"raw":0.7547,"skill":0.6453,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.9508,"skill":0.9016,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5784,"skill":0.1568,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.1287,"skill":0.1219,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.6146,"skill":0.2292,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8022,"skill":0.7363,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4694,"skill":0.2925,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.5489,"skill":0.4539,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.5959,"headline":false},{"track":"GSM8K-10choice","score":0.5019,"headline":false}]},"31":{"raw":0.1732,"skill":0.0995,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4922,"skill":0.4256,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.5351,"skill":0.4169,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.5324,"skill":0.3985,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3969,"headline":false},{"track":"A · English","score":0.5324,"headline":true},{"track":"C · Arabic pairs","score":0.72,"headline":false},{"track":"C · English pairs","score":0.935,"headline":false}]},"41":{"raw":0.6081,"skill":0.4121,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.5368,"skill":0.2652,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6178,"skill":0.2356,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.395,"skill":0.1219,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.7435,"skill":0.487,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.5727,"skill":0.5194,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6797,"skill":0.5357,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.7755,"skill":0.7007,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.6875,"skill":0.6094,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.8026,"skill":0.5907,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.7117,"skill":0.4234,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5944,"skill":0.3552,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2752,"skill":0.2656,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1198,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.6355,"skill":0.454,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.7451,"skill":0.4902,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.9046,"skill":0.8728,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.2238,"skill":0.199,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8032,"skill":0.7105,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7522,"skill":0.5179,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.7517,"skill":0.7485,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.8128,"skill":0.8117,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.8406,"skill":0.8375,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.4773,"skill":0.4773,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.1484,"skill":0.1484,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author DecisionEngine over vLLM 0.29.0 OpenAI server, Qwen3.8-27B-FP8, fp8 KV, prefix caching, CUDA graphs, max-model-len 262144 (board lift), gpu util 0.9"},{"engine":"jevk5","name":"JevK5","long_name":"JevK5 Qwen3.5-4B LoRA","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":8411558400,"model_url":"https://huggingface.co/alibiserikbay/JevK5","code_url":"https://github.com/allebee/jevk5","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"alibiserikbay/JevK5","served_checkpoint":null},"completed":120340,"counts":{"ok":120340},"lifted":true,"calibration":{"acc":0.6475,"conf":0.6742,"ece":0.0268,"brier":0.4591,"over95":0.0055,"n":35093,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0023,"conf":0.0875,"acc":0.0664},{"lo":0.1,"w":0.0343,"conf":0.1567,"acc":0.088},{"lo":0.2,"w":0.0361,"conf":0.2467,"acc":0.1751},{"lo":0.3,"w":0.0639,"conf":0.3512,"acc":0.2823},{"lo":0.4,"w":0.0757,"conf":0.4509,"acc":0.3824},{"lo":0.5,"w":0.1489,"conf":0.5467,"acc":0.5456},{"lo":0.6,"w":0.1424,"conf":0.6469,"acc":0.6422},{"lo":0.7,"w":0.124,"conf":0.7492,"acc":0.7103},{"lo":0.8,"w":0.1642,"conf":0.8497,"acc":0.8333},{"lo":0.9,"w":0.2081,"conf":0.9474,"acc":0.9291}],"areas":{"knowledge":{"ece":0.0453,"acc":0.5718,"conf":0.6093},"language":{"ece":0.0149,"acc":0.7447,"conf":0.7399},"retrieval":{"ece":0.0487,"acc":0.6709,"conf":0.6867},"tools":{"ece":0.0245,"acc":0.8182,"conf":0.8216},"arts":{"ece":0.0828,"acc":0.5177,"conf":0.6005}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":22.0,"p95":226.3,"mean":89.8},"results":{"1":{"score":0.8501,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":62.7},"2":{"score":0.5836,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":712.5},"3":{"score":0.685,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.6918,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":124.7},"5":{"score":0.7466,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":218.2},"6":{"score":0.7864,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":219.3},"9":{"score":0.125,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6298,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":55.0},"11":{"score":0.669,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5513,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":14.8},"20":{"score":0.8404,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.8},"21":{"score":0.6088,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":12.6},"22":{"score":0.016,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1092.5},"23":{"score":0.617,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":24.5},"24":{"score":0.7466,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":13.9},"25":{"score":0.4286,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9739,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":13.7},"27":{"score":0.9249,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":13.8},"28":{"score":0.7111,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":12.5},"29":{"score":0.8352,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":16.4},"30":{"score":0.4246,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":16.3},"31":{"score":0.13,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":92.1},"32":{"score":0.5186,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.283,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":202.4},"36":{"score":0.4286,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4081,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.1},"38":{"score":0.1508,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":883.1},"39":{"score":0.877,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4582,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":12.9,"tracks":[{"track":"A · Arabic","score":0.3427,"headline":false},{"track":"A · English","score":0.4582,"headline":true},{"track":"C · Arabic pairs","score":0.665,"headline":false},{"track":"C · English pairs","score":0.91,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4568,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":15.0},"42":{"score":0.7614,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":25.8},"43":{"score":0.4895,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5938,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.8},"45":{"score":0.1357,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.1985,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":37.3},"50":{"score":0.4648,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":37.8},"56":{"score":0.7,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":146.5,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.4914,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":16.4,"edition_added":"v0.2","wrong_rows":980,"contamination":"980 contaminated MMLU-Pro test items (in its training data) count as wrong","jev":0.827},"58":{"score":0.5983,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":15.4,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.5487,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":35.2,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.5549,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.645,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":37.6,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.6222,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":23.6,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":53.68,"balanced_skill":38.81,"breadth_skill":37.49},"frozen_scores":{"balanced_raw":40.5,"balanced_skill":25.98,"breadth_skill":22.38},"categories":[{"id":"knowledge","raw":0.4097,"skill":0.235,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.6165,"skill":0.4329,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.6016,"skill":0.4615,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.574,"skill":0.5212,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4615,"skill":0.2781,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.8501,"skill":0.7976,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.5836,"skill":0.5191,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7864,"skill":0.4975,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7677,"headline":false},{"track":"RouterBench-5shot","score":0.805,"headline":false}]},"11":{"raw":0.669,"skill":0.5214,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.8436,"skill":0.6872,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6088,"skill":0.2177,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0166,"skill":0.0089,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5872,"skill":0.1745,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.761,"skill":0.6813,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4286,"skill":0.2381,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.4246,"skill":0.2996,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.4503,"headline":false},{"track":"GSM8K-10choice","score":0.3988,"headline":false}]},"31":{"raw":0.13,"skill":0.0524,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4286,"skill":0.3536,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.4081,"skill":0.2576,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.4582,"skill":0.303,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3427,"headline":false},{"track":"A · English","score":0.4582,"headline":true},{"track":"C · Arabic pairs","score":0.665,"headline":false},{"track":"C · English pairs","score":0.91,"headline":false}]},"41":{"raw":0.4568,"skill":0.1852,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4895,"skill":0.19,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5938,"skill":0.1876,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4648,"skill":0.2232,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.7,"skill":0.4,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.4914,"skill":0.428,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.5983,"skill":0.4177,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.5487,"skill":0.3983,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.5549,"skill":0.4436,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.645,"skill":0.2639,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.6222,"skill":0.2444,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5186,"skill":0.2347,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.283,"skill":0.2735,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1357,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.5513,"skill":0.3279,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.7111,"skill":0.4222,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.8352,"skill":0.7803,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.1508,"skill":0.1236,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.877,"skill":0.8191,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7614,"skill":0.5358,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.6918,"skill":0.6878,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.7466,"skill":0.7451,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.685,"skill":0.6789,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.125,"skill":0.125,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.206,"skill":0.206,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"JevK5 0.2.2 lifted (JEVK5_LIFT=1, knockout readout), CUDA graphs on (package default JevK5(graphs=True)), fla kernels; venv torch 2.14.0+cu130 / transformers 5.17.0 / fla 0.5.2","contamination":[{"catalog_id":57,"benchmark":"MMLU-Pro","wrong":980,"flag":"980 contaminated MMLU-Pro test items (in its training data) count as wrong"}],"new_in_edition":true},{"engine":"jobe","name":"Jobe","long_name":"Jobe Qwen3.5-4B","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/MantisShrimpdev/jobe","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":null,"served_checkpoint":"Qwen/Qwen3.5-4B"},"completed":120340,"counts":{"ok":120340},"lifted":true,"calibration":{"acc":0.6065,"conf":0.6499,"ece":0.1232,"brier":0.536,"over95":0.0226,"n":35093,"benchmarks":32,"rel":[{"lo":0.0,"w":0.099,"conf":0.0569,"acc":0.2647},{"lo":0.1,"w":0.0443,"conf":0.1358,"acc":0.4986},{"lo":0.2,"w":0.0211,"conf":0.2494,"acc":0.4015},{"lo":0.3,"w":0.0406,"conf":0.3549,"acc":0.2825},{"lo":0.4,"w":0.0547,"conf":0.4491,"acc":0.3432},{"lo":0.5,"w":0.1407,"conf":0.5488,"acc":0.5012},{"lo":0.6,"w":0.0933,"conf":0.6498,"acc":0.5601},{"lo":0.7,"w":0.1043,"conf":0.7512,"acc":0.6157},{"lo":0.8,"w":0.1017,"conf":0.8525,"acc":0.6877},{"lo":0.9,"w":0.3003,"conf":0.9695,"acc":0.8742}],"areas":{"knowledge":{"ece":0.0978,"acc":0.5389,"conf":0.6363},"language":{"ece":0.1308,"acc":0.6964,"conf":0.8237},"retrieval":{"ece":0.3404,"acc":0.6107,"conf":0.4449},"tools":{"ece":0.1215,"acc":0.7509,"conf":0.6344},"arts":{"ece":0.046,"acc":0.5032,"conf":0.5405}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":53.5,"p95":2910.3,"mean":322.6},"results":{"1":{"score":0.9298,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":108.8},"2":{"score":0.5937,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1662.3},"3":{"score":0.4606,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.6237,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":1385.1},"5":{"score":0.5441,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":2931.6},"6":{"score":0.7952,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":219.4},"9":{"score":0.0795,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5657,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":58.5},"11":{"score":0.5924,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5453,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":50.9},"20":{"score":0.8516,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":50.6},"21":{"score":0.6046,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":50.7},"22":{"score":0.0515,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":3239.2},"23":{"score":0.5688,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":51.7},"24":{"score":0.7127,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":51.0},"25":{"score":0.3724,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9684,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":51.0},"27":{"score":0.9172,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":51.6},"28":{"score":0.6606,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":50.2},"29":{"score":0.8351,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":52.2},"30":{"score":0.3116,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":53.8},"31":{"score":0.134,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":560.0},"32":{"score":0.5439,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2018,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":415.9},"36":{"score":0.4118,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3581,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":53.5},"38":{"score":0.134,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":3228.1},"39":{"score":0.6581,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.3896,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":51.2,"tracks":[{"track":"A · Arabic","score":0.2774,"headline":false},{"track":"A · English","score":0.3896,"headline":true},{"track":"C · Arabic pairs","score":0.65,"headline":false},{"track":"C · English pairs","score":0.825,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4366,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":51.6},"42":{"score":0.7185,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":53.3},"43":{"score":0.3912,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5638,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":50.9},"45":{"score":0.1138,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2248,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":54.6},"50":{"score":0.4427,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":58.4},"56":{"score":0.6,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":458.3,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.4364,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":54.9,"edition_added":"v0.2","jev":0.827},"58":{"score":0.5733,"metric":"accuracy","requests":5507,"answered":5332,"unsupported":175,"errors":0,"median_ms":51.9,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.549,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":57.1,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.5814,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.5763,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":54.9,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.6165,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":53.4,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":48.67,"balanced_skill":32.35,"breadth_skill":30.98},"frozen_scores":{"balanced_raw":38.61,"balanced_skill":23.18,"breadth_skill":20.04},"categories":[{"id":"knowledge","raw":0.3629,"skill":0.1773,"coverage":0.9965,"pending":0,"n":10},{"id":"language","raw":0.5618,"skill":0.3454,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.5258,"skill":0.3669,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.5353,"skill":0.4881,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4452,"skill":0.2574,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9298,"skill":0.9052,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.5937,"skill":0.5308,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7953,"skill":0.5167,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7834,"headline":false},{"track":"RouterBench-5shot","score":0.8071,"headline":false}]},"11":{"raw":0.5924,"skill":0.4105,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.8512,"skill":0.7024,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6046,"skill":0.2093,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0494,"skill":0.042,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5552,"skill":0.1104,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7242,"skill":0.6322,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3724,"skill":0.1633,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.3116,"skill":0.1686,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.4018,"headline":false},{"track":"GSM8K-10choice","score":0.2214,"headline":false}]},"31":{"raw":0.134,"skill":0.0568,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4118,"skill":0.3346,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.3581,"skill":0.1948,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.3896,"skill":0.2147,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2774,"headline":false},{"track":"A · English","score":0.3896,"headline":true},{"track":"C · Arabic pairs","score":0.65,"headline":false},{"track":"C · English pairs","score":0.825,"headline":false}]},"41":{"raw":0.4366,"skill":0.1549,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3912,"skill":0.0341,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5638,"skill":0.1276,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4427,"skill":0.1912,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.6,"skill":0.2,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.4364,"skill":0.3661,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.5551,"skill":0.3551,"coverage":0.9682,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.549,"skill":0.3987,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.5814,"skill":0.4768,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.5763,"skill":0.1215,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.6165,"skill":0.233,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5439,"skill":0.2749,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2018,"skill":0.1912,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1138,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.5453,"skill":0.3189,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.6606,"skill":0.3212,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.8351,"skill":0.7801,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.134,"skill":0.1063,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.6581,"skill":0.4971,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7185,"skill":0.4523,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.6237,"skill":0.6189,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.5441,"skill":0.5413,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.4606,"skill":0.4502,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0795,"skill":0.0795,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.1008,"skill":0.1008,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author jobe.score per question with SDPA attention (published default eager; sdpa is faster: median 53.5 vs 55.9 ms, p95 2910 vs 3088 ms); JOBE_LIFT=1 board lifts; venv-q torch 2.14.0+cu130. The eager run is kept as latency-v1-full.eager","new_in_edition":true},{"engine":"joshua-diffusion-full","name":"JoshuaSP diffusiongemma (open-jev)","long_name":"JoshuaSP diffgemma","variant":null,"meta":{"kind":"inference technique","technique":"diffusion","technique_name":"diffusion","served_params":25823778864,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/JoshuaSP/open-jev","base_model":"google/diffusiongemma-26B-A4B-it","base_url":"https://huggingface.co/google/diffusiongemma-26B-A4B-it","weights_repo":null,"served_checkpoint":"google/diffusiongemma-26B-A4B-it"},"completed":120340,"counts":{"ok":120025},"lifted":false,"calibration":{"acc":0.6805,"conf":0.896,"ece":0.2155,"brier":0.5241,"over95":0.1366,"n":72589,"benchmarks":32,"rel":[{"lo":0.0,"w":0.002,"conf":0.0894,"acc":0.0},{"lo":0.1,"w":0.0148,"conf":0.1478,"acc":0.0},{"lo":0.2,"w":0.0121,"conf":0.2486,"acc":0.0881},{"lo":0.3,"w":0.0154,"conf":0.3542,"acc":0.1417},{"lo":0.4,"w":0.0191,"conf":0.4495,"acc":0.1841},{"lo":0.5,"w":0.037,"conf":0.5496,"acc":0.4329},{"lo":0.6,"w":0.0384,"conf":0.6459,"acc":0.4039},{"lo":0.7,"w":0.0435,"conf":0.7526,"acc":0.4219},{"lo":0.8,"w":0.067,"conf":0.8528,"acc":0.5294},{"lo":0.9,"w":0.7507,"conf":0.9878,"acc":0.7837}],"areas":{"knowledge":{"ece":0.2513,"acc":0.6139,"conf":0.8651},"language":{"ece":0.1913,"acc":0.7806,"conf":0.9719},"retrieval":{"ece":0.2289,"acc":0.6582,"conf":0.886},"tools":{"ece":0.0448,"acc":0.9328,"conf":0.9743},"arts":{"ece":0.2749,"acc":0.513,"conf":0.788}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":260.4,"p95":537.3,"mean":337.8},"results":{"1":{"score":0.9652,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":272.9},"2":{"score":0.6121,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1603.1},"3":{"score":0.8366,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7428,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":316.5},"5":{"score":0.8349,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":465.5},"6":{"score":0.79,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":488.9},"9":{"score":0.4205,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4065,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":293.7},"11":{"score":0.7596,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6637,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":255.1},"20":{"score":0.8694,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":256.6},"21":{"score":0.6298,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":249.9},"22":{"score":0.025,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1205.3},"23":{"score":0.5824,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":259.3},"24":{"score":0.7926,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":250.8},"25":{"score":0.449,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9823,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":248.2},"27":{"score":0.9445,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":249.0},"28":{"score":0.7364,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":249.1},"29":{"score":0.8328,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":259.4},"30":{"score":0.5027,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":251.0},"31":{"score":0.1416,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":277.6},"32":{"score":0.6117,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2745,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":291.8},"36":{"score":0.4294,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5337,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":259.4},"38":{"score":0.2449,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1667.8},"39":{"score":0.8904,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.66,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":246.5,"tracks":[{"track":"A · Arabic","score":0.49,"headline":false},{"track":"A · English","score":0.66,"headline":true},{"track":"C · Arabic pairs","score":0.8,"headline":false},{"track":"C · English pairs","score":0.945,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.5569,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":256.6},"42":{"score":0.7843,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":267.1},"43":{"score":0.6474,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6778,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":254.6},"45":{"score":0.0978,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2959,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":276.2},"50":{"score":0.4499,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":273.0},"56":{"score":0.8535,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":283.3,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.5686,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":252.0,"edition_added":"v0.2","jev":0.827},"58":{"score":0.7067,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":247.5,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.7544,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":269.6,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.6364,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.7043,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":271.5,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.7085,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":261.8,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":61.28,"balanced_skill":49.47,"breadth_skill":47.5},"frozen_scores":{"balanced_raw":44.55,"balanced_skill":31.97,"breadth_skill":26.82},"categories":[{"id":"knowledge","raw":0.4669,"skill":0.3272,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.6868,"skill":0.5351,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.6825,"skill":0.5837,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.7313,"skill":0.7016,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4429,"skill":0.2665,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9652,"skill":0.953,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6121,"skill":0.5521,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.79,"skill":0.5041,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.779,"headline":false},{"track":"RouterBench-5shot","score":0.8011,"headline":false}]},"11":{"raw":0.7596,"skill":0.6523,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.8729,"skill":0.7459,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6298,"skill":0.2595,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0263,"skill":0.0187,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5739,"skill":0.1477,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7954,"skill":0.7272,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.449,"skill":0.2653,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.5027,"skill":0.3995,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.5686,"headline":false},{"track":"GSM8K-10choice","score":0.4367,"headline":false}]},"31":{"raw":0.1416,"skill":0.0651,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4294,"skill":0.3545,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.5337,"skill":0.4152,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.66,"skill":0.5626,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.49,"headline":false},{"track":"A · English","score":0.66,"headline":true},{"track":"C · Arabic pairs","score":0.8,"headline":false},{"track":"C · English pairs","score":0.945,"headline":false}]},"41":{"raw":0.5569,"skill":0.3353,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.6474,"skill":0.4405,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6778,"skill":0.3556,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4499,"skill":0.2016,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.8535,"skill":0.707,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.5686,"skill":0.5148,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.7067,"skill":0.5749,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.7544,"skill":0.6725,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.6364,"skill":0.5455,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.7043,"skill":0.3869,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.7085,"skill":0.417,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.6117,"skill":0.3827,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2745,"skill":0.2649,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.0978,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.6637,"skill":0.4963,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.7364,"skill":0.4728,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.8328,"skill":0.7771,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.2449,"skill":0.2207,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8904,"skill":0.8388,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7843,"skill":0.5804,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.7428,"skill":0.7395,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.8349,"skill":0.8339,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.8366,"skill":0.8335,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.4205,"skill":0.4205,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"open-jev[inference] diffusiongemma engine, bf16, chunked-expert path as board; venv-joshua"},{"engine":"kev-0.8b-raised","name":"Kev 0.8B","long_name":"Kev 0.8B","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":873438784,"params_estimated":false,"params_from_name":false,"trained_bytes":45441727,"model_url":"https://huggingface.co/jaredpalmer/kev-0.8b","code_url":"https://github.com/jaredpalmer/kev","base_model":"Qwen/Qwen3.5-0.8B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B-Base","weights_repo":"jaredpalmer/kev-0.8b","served_checkpoint":"jaredpalmer/kev-0.8b"},"completed":120340,"counts":{"ok":120340},"lifted":false,"calibration":{"acc":0.4842,"conf":0.5363,"ece":0.0743,"brier":0.6347,"over95":0.0038,"n":72594,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0623,"conf":0.0427,"acc":0.0714},{"lo":0.1,"w":0.0563,"conf":0.1391,"acc":0.2258},{"lo":0.2,"w":0.0462,"conf":0.2587,"acc":0.2172},{"lo":0.3,"w":0.1142,"conf":0.3517,"acc":0.3847},{"lo":0.4,"w":0.095,"conf":0.4494,"acc":0.4564},{"lo":0.5,"w":0.2086,"conf":0.5478,"acc":0.5181},{"lo":0.6,"w":0.1457,"conf":0.6468,"acc":0.5718},{"lo":0.7,"w":0.125,"conf":0.7491,"acc":0.598},{"lo":0.8,"w":0.093,"conf":0.8453,"acc":0.689},{"lo":0.9,"w":0.0537,"conf":0.9347,"acc":0.7352}],"areas":{"knowledge":{"ece":0.0814,"acc":0.409,"conf":0.4781},"language":{"ece":0.1084,"acc":0.5509,"conf":0.6541},"retrieval":{"ece":0.1392,"acc":0.5533,"conf":0.4649},"tools":{"ece":0.0603,"acc":0.65,"conf":0.6243},"arts":{"ece":0.0817,"acc":0.3749,"conf":0.4566}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":41.6,"p95":95.9,"mean":102.9},"results":{"1":{"score":0.6405,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":87.0},"2":{"score":0.5238,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":198.3},"3":{"score":0.4291,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7622,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":44.5},"5":{"score":0.0922,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":47.3},"6":{"score":0.7036,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":90.0},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5354,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":87.1},"11":{"score":0.3509,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.3288,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":40.7},"20":{"score":0.5186,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":40.6},"21":{"score":0.519,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":40.0},"22":{"score":0.0145,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":129.5},"23":{"score":0.5672,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":85.8},"24":{"score":0.4494,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":40.2},"25":{"score":0.3061,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.8161,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":40.0},"27":{"score":0.6092,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":39.9},"28":{"score":0.5162,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":39.9},"29":{"score":0.4229,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":41.2},"30":{"score":0.1478,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":40.7},"31":{"score":0.0918,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":42.7},"32":{"score":0.4907,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0115,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":67.7},"36":{"score":0.2853,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.209,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":41.5},"38":{"score":0.0777,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":356.2},"39":{"score":0.7012,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.1585,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":39.9,"tracks":[{"track":"A · Arabic","score":0.0733,"headline":false},{"track":"A · English","score":0.1585,"headline":true},{"track":"C · Arabic pairs","score":0.46,"headline":false},{"track":"C · English pairs","score":0.325,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3962,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":40.7},"42":{"score":0.5482,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":86.0},"43":{"score":0.3456,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5218,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":40.6},"45":{"score":0.1377,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2366,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":86.1},"50":{"score":0.3168,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":86.3},"56":{"score":0.5005,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":86.0,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.216,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":41.1,"edition_added":"v0.2","jev":0.827},"58":{"score":0.4137,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":40.6,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.3486,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":86.7,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.2917,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.4002,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":86.6,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.505,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":63.4,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":35.36,"balanced_skill":14.6,"breadth_skill":13.09},"frozen_scores":{"balanced_raw":28.62,"balanced_skill":8.89,"breadth_skill":7.34},"categories":[{"id":"knowledge","raw":0.2683,"skill":0.0608,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.3919,"skill":0.1133,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.3912,"skill":0.1914,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.3992,"skill":0.315,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.3158,"skill":0.0511,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.6405,"skill":0.5147,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.5238,"skill":0.4501,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7036,"skill":0.305,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.6693,"headline":false},{"track":"RouterBench-5shot","score":0.7379,"headline":false}]},"11":{"raw":0.3509,"skill":0.0614,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.5112,"skill":0.0223,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.519,"skill":0.0381,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0138,"skill":0.0061,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.557,"skill":0.114,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.46,"skill":0.28,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3061,"skill":0.0748,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.1478,"skill":0.0,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.2168,"headline":false},{"track":"GSM8K-10choice","score":0.0788,"headline":false}]},"31":{"raw":0.0918,"skill":0.0108,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.2853,"skill":0.1914,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.209,"skill":0.0078,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.1585,"skill":0.0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0733,"headline":false},{"track":"A · English","score":0.1585,"headline":true},{"track":"C · Arabic pairs","score":0.46,"headline":false},{"track":"C · English pairs","score":0.325,"headline":false}]},"41":{"raw":0.3962,"skill":0.0944,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3456,"skill":0.0,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5218,"skill":0.0436,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3168,"skill":0.0085,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5005,"skill":0.001,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.216,"skill":0.1182,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.4137,"skill":0.1502,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.3486,"skill":0.1315,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.2917,"skill":0.1146,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.4002,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.505,"skill":0.01,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.4907,"skill":0.1903,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0115,"skill":0.0,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1377,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.3288,"skill":0.0,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5162,"skill":0.0324,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.4229,"skill":0.2305,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0777,"skill":0.0482,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.7012,"skill":0.5605,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.5482,"skill":0.121,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.7622,"skill":0.7591,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.0922,"skill":0.0867,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.4291,"skill":0.4181,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0536,"skill":0.0536,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"native Kev serving: merged bf16 backbone, fp32 pointer head, sdpa, native prefix cache 4, FLA kernels, head.pt T=2.406 applied as on the board; venv-kev torch 2.8.0+cu128"},{"engine":"kev-4b-raised","name":"Kev 4B","long_name":"Kev 4B","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":135172799,"model_url":"https://huggingface.co/jaredpalmer/kev-4b","code_url":"https://github.com/jaredpalmer/kev","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"jaredpalmer/kev-4b","served_checkpoint":"jaredpalmer/kev-4b"},"completed":120340,"counts":{"ok":120340},"lifted":false,"calibration":{"acc":0.6161,"conf":0.7917,"ece":0.1756,"brier":0.545,"over95":0.0717,"n":72594,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0123,"conf":0.0808,"acc":0.0527},{"lo":0.1,"w":0.0158,"conf":0.1418,"acc":0.1444},{"lo":0.2,"w":0.0172,"conf":0.2604,"acc":0.1604},{"lo":0.3,"w":0.0396,"conf":0.3554,"acc":0.1695},{"lo":0.4,"w":0.0528,"conf":0.4513,"acc":0.2605},{"lo":0.5,"w":0.0863,"conf":0.5466,"acc":0.4282},{"lo":0.6,"w":0.08,"conf":0.6493,"acc":0.4725},{"lo":0.7,"w":0.0861,"conf":0.7496,"acc":0.5142},{"lo":0.8,"w":0.1154,"conf":0.8536,"acc":0.6469},{"lo":0.9,"w":0.4947,"conf":0.9784,"acc":0.8012}],"areas":{"knowledge":{"ece":0.1592,"acc":0.5503,"conf":0.7089},"language":{"ece":0.1737,"acc":0.7049,"conf":0.8772},"retrieval":{"ece":0.1279,"acc":0.7092,"conf":0.8362},"tools":{"ece":0.1915,"acc":0.7713,"conf":0.9571},"arts":{"ece":0.2512,"acc":0.4336,"conf":0.6849}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":52.1,"p95":141.1,"mean":100.0},"results":{"1":{"score":0.9227,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":110.1},"2":{"score":0.6217,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":732.0},"3":{"score":0.5118,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8224,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":55.0},"5":{"score":0.7936,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":62.5},"6":{"score":0.8007,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":126.8},"9":{"score":0.3068,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5693,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":109.5},"11":{"score":0.6224,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.4621,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":51.4},"20":{"score":0.59,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":51.4},"21":{"score":0.5293,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":50.9},"22":{"score":0.0115,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":310.8},"23":{"score":0.5884,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":109.1},"24":{"score":0.6932,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":51.5},"25":{"score":0.3878,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.971,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":51.2},"27":{"score":0.8985,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":51.2},"28":{"score":0.6851,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":50.7},"29":{"score":0.7537,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":51.7},"30":{"score":0.4011,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":51.6},"31":{"score":0.0982,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":53.8},"32":{"score":0.5824,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2685,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":100.3},"36":{"score":0.3968,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3848,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":52.0},"38":{"score":0.1226,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":439.9},"39":{"score":0.8179,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4231,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":51.0,"tracks":[{"track":"A · Arabic","score":0.2552,"headline":false},{"track":"A · English","score":0.4231,"headline":true},{"track":"C · Arabic pairs","score":0.495,"headline":false},{"track":"C · English pairs","score":0.755,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.5094,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":51.6},"42":{"score":0.7202,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":108.9},"43":{"score":0.4684,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6022,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":51.6},"45":{"score":0.1118,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.1859,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":108.4},"50":{"score":0.4177,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":109.8},"56":{"score":0.5005,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":118.5,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.4632,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":51.8,"edition_added":"v0.2","jev":0.827},"58":{"score":0.6098,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":51.5,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.4682,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":109.2,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.5549,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.3303,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":108.9,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5797,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":80.3,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":50.99,"balanced_skill":34.64,"breadth_skill":33.18},"frozen_scores":{"balanced_raw":38.07,"balanced_skill":21.74,"breadth_skill":17.87},"categories":[{"id":"knowledge","raw":0.3989,"skill":0.2286,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.5497,"skill":0.353,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.5879,"skill":0.4098,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.5774,"skill":0.5256,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4142,"skill":0.1792,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9227,"skill":0.8956,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6217,"skill":0.5631,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.8007,"skill":0.53,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7879,"headline":false},{"track":"RouterBench-5shot","score":0.8135,"headline":false}]},"11":{"raw":0.6224,"skill":0.4539,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.5953,"skill":0.1905,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5293,"skill":0.0586,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0114,"skill":0.0037,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5657,"skill":0.1314,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7,"skill":0.6,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3878,"skill":0.1837,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.4011,"skill":0.2783,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.4943,"headline":false},{"track":"GSM8K-10choice","score":0.3078,"headline":false}]},"31":{"raw":0.0982,"skill":0.0178,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.3968,"skill":0.3176,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.3848,"skill":0.2283,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.4231,"skill":0.2578,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2552,"headline":false},{"track":"A · English","score":0.4231,"headline":true},{"track":"C · Arabic pairs","score":0.495,"headline":false},{"track":"C · English pairs","score":0.755,"headline":false}]},"41":{"raw":0.5094,"skill":0.2641,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4684,"skill":0.1566,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6022,"skill":0.2044,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4177,"skill":0.1548,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5005,"skill":0.001,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.4632,"skill":0.3962,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6098,"skill":0.4344,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.4682,"skill":0.2909,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.5549,"skill":0.4436,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.3303,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5797,"skill":0.1594,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5824,"skill":0.3361,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2685,"skill":0.2588,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1118,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.4621,"skill":0.1943,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.6851,"skill":0.3702,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.7537,"skill":0.6716,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.1226,"skill":0.0945,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8179,"skill":0.7322,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7202,"skill":0.4556,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.8224,"skill":0.8201,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.7936,"skill":0.7924,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.5118,"skill":0.5024,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.3068,"skill":0.3068,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.2564,"skill":0.2564,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"native Kev serving at the board checkpoint (raw probabilities, no temperature, as on the board): merged bf16 backbone, fp32 pointer head, sdpa, native prefix cache 4, FLA kernels; venv-kev torch 2.8.0+cu128. A first run on the newer checkpoint (kev-models.json head) was set aside."},{"engine":"kev-9b-raised","name":"Kev 9B","long_name":"Kev 9B","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":9653104368,"params_estimated":false,"params_from_name":false,"trained_bytes":181572591,"model_url":"https://huggingface.co/jaredpalmer/kev-9b","code_url":"https://github.com/jaredpalmer/kev","base_model":"Qwen/Qwen3.5-9B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B-Base","weights_repo":"jaredpalmer/kev-9b","served_checkpoint":"jaredpalmer/kev-9b"},"completed":120340,"counts":{"ok":120340},"lifted":false,"calibration":{"acc":0.6521,"conf":0.7899,"ece":0.1378,"brier":0.487,"over95":0.0548,"n":72594,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0133,"conf":0.0886,"acc":0.0389},{"lo":0.1,"w":0.0322,"conf":0.138,"acc":0.0733},{"lo":0.2,"w":0.0163,"conf":0.2498,"acc":0.1809},{"lo":0.3,"w":0.0303,"conf":0.3541,"acc":0.2577},{"lo":0.4,"w":0.0371,"conf":0.4538,"acc":0.3623},{"lo":0.5,"w":0.0938,"conf":0.5482,"acc":0.4134},{"lo":0.6,"w":0.0732,"conf":0.65,"acc":0.4996},{"lo":0.7,"w":0.0892,"conf":0.7519,"acc":0.5815},{"lo":0.8,"w":0.1137,"conf":0.8548,"acc":0.6541},{"lo":0.9,"w":0.5008,"conf":0.9772,"acc":0.8454}],"areas":{"knowledge":{"ece":0.1544,"acc":0.5804,"conf":0.7348},"language":{"ece":0.1256,"acc":0.7604,"conf":0.886},"retrieval":{"ece":0.116,"acc":0.7455,"conf":0.8567},"tools":{"ece":0.1603,"acc":0.7877,"conf":0.9434},"arts":{"ece":0.1379,"acc":0.4586,"conf":0.5927}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":51.4,"p95":187.9,"mean":118.2},"results":{"1":{"score":0.9451,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":108.7},"2":{"score":0.6426,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1133.5},"3":{"score":0.563,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8483,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":61.3},"5":{"score":0.7903,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":98.3},"6":{"score":0.8005,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":169.7},"9":{"score":0.25,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6398,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":121.2},"11":{"score":0.5777,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5626,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":50.3},"20":{"score":0.67,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":49.9},"21":{"score":0.5575,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":49.9},"22":{"score":0.1075,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":442.6},"23":{"score":0.5632,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":106.2},"24":{"score":0.7529,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":50.0},"25":{"score":0.3878,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9773,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":49.9},"27":{"score":0.9369,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":50.2},"28":{"score":0.7316,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":49.4},"29":{"score":0.8192,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":50.4},"30":{"score":0.4871,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":50.1},"31":{"score":0.1116,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":57.6},"32":{"score":0.5785,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2673,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":119.4},"36":{"score":0.3853,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4922,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":51.1},"38":{"score":0.1831,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":557.5},"39":{"score":0.8838,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4708,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":49.9,"tracks":[{"track":"A · Arabic","score":0.3118,"headline":false},{"track":"A · English","score":0.4708,"headline":true},{"track":"C · Arabic pairs","score":0.545,"headline":false},{"track":"C · English pairs","score":0.705,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.5542,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":50.2},"42":{"score":0.7486,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":106.2},"43":{"score":0.5123,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.62,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":50.1},"45":{"score":0.1038,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.176,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":106.0},"50":{"score":0.3944,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":106.9},"56":{"score":0.5075,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":150.9,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.5113,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":50.4,"edition_added":"v0.2","jev":0.827},"58":{"score":0.6517,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":50.1,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.4962,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":106.8,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.5814,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.4618,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":106.4,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.588,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":78.6,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":54.04,"balanced_skill":38.48,"breadth_skill":37.22},"frozen_scores":{"balanced_raw":40.26,"balanced_skill":24.78,"breadth_skill":20.66},"categories":[{"id":"knowledge","raw":0.4224,"skill":0.2616,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.6053,"skill":0.4167,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.6085,"skill":0.4367,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.5923,"skill":0.5448,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4466,"skill":0.2242,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9451,"skill":0.9259,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6426,"skill":0.5873,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.8005,"skill":0.5296,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7873,"headline":false},{"track":"RouterBench-5shot","score":0.8137,"headline":false}]},"11":{"raw":0.5777,"skill":0.3892,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.6722,"skill":0.3444,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5575,"skill":0.1149,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.1012,"skill":0.0942,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5539,"skill":0.1079,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7619,"skill":0.6825,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3878,"skill":0.1837,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.4871,"skill":0.384,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.5853,"headline":false},{"track":"GSM8K-10choice","score":0.3889,"headline":false}]},"31":{"raw":0.1116,"skill":0.0324,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.3853,"skill":0.3046,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.4922,"skill":0.363,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.4708,"skill":0.3192,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3118,"headline":false},{"track":"A · English","score":0.4708,"headline":true},{"track":"C · Arabic pairs","score":0.545,"headline":false},{"track":"C · English pairs","score":0.705,"headline":false}]},"41":{"raw":0.5542,"skill":0.3314,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.5123,"skill":0.2262,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.62,"skill":0.24,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3944,"skill":0.1211,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5075,"skill":0.015,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.5113,"skill":0.4503,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6517,"skill":0.4951,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.4962,"skill":0.3283,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.5814,"skill":0.4768,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.4618,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.588,"skill":0.176,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5785,"skill":0.3299,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2673,"skill":0.2576,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1038,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.5626,"skill":0.3448,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.7316,"skill":0.4632,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.8192,"skill":0.7589,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.1831,"skill":0.157,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8838,"skill":0.8291,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7486,"skill":0.5109,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.8483,"skill":0.8463,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.7903,"skill":0.789,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.563,"skill":0.5546,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.25,"skill":0.25,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.296,"skill":0.296,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"native Kev serving at the board checkpoint (raw probabilities, no temperature, as on the board): merged bf16 backbone, fp32 pointer head, sdpa, native prefix cache 4, FLA kernels; venv-kev torch 2.8.0+cu128. A first run on the newer checkpoint (kev-models.json head) was set aside."},{"engine":"laya-1e28ac20","name":"Laya","long_name":"Laya","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":421293830,"params_estimated":false,"params_from_name":false,"trained_bytes":2329053944,"model_url":"https://huggingface.co/convaiinnovations/laya","code_url":"https://github.com/NandhaKishorM/laya","base_model":null,"base_url":null,"weights_repo":"convaiinnovations/laya","served_checkpoint":null},"completed":120340,"counts":{"ok":119906,"unsupported":320},"lifted":false,"calibration":{"acc":0.377,"conf":0.5172,"ece":0.1402,"brier":0.7211,"over95":0.0116,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0485,"conf":0.0771,"acc":0.0455},{"lo":0.1,"w":0.0375,"conf":0.1433,"acc":0.1199},{"lo":0.2,"w":0.0673,"conf":0.2665,"acc":0.1752},{"lo":0.3,"w":0.2026,"conf":0.3479,"acc":0.2867},{"lo":0.4,"w":0.108,"conf":0.4446,"acc":0.3428},{"lo":0.5,"w":0.2027,"conf":0.5458,"acc":0.4187},{"lo":0.6,"w":0.1071,"conf":0.6457,"acc":0.4688},{"lo":0.7,"w":0.0765,"conf":0.7488,"acc":0.503},{"lo":0.8,"w":0.0755,"conf":0.8519,"acc":0.5633},{"lo":0.9,"w":0.0742,"conf":0.9455,"acc":0.6364}],"areas":{"knowledge":{"ece":0.1577,"acc":0.3142,"conf":0.4719},"language":{"ece":0.1218,"acc":0.545,"conf":0.6648},"retrieval":{"ece":0.097,"acc":0.2769,"conf":0.3529},"tools":{"ece":0.2475,"acc":0.2407,"conf":0.475},"arts":{"ece":0.1334,"acc":0.3744,"conf":0.5078}}},"gaps":{"share":0.0027,"reasons":[{"reason":"instructions longer than its native budget (no truncation allowed)","share":0.0026},{"reason":"prompt longer than its context window (no truncation allowed)","share":0.0},{"reason":"answer options longer than its per-option budget (no truncation allowed)","share":0.0}]},"latency":{"median":5.8,"p95":222.5,"mean":32.7},"results":{"1":{"score":0.3813,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":38.7},"2":{"score":0.1278,"jev":0.6535,"metric":"nDCG@10","requests":685,"answered":680,"unsupported":5,"errors":0,"median_ms":null},"3":{"score":0.1146,"jev":0.8834,"metric":"accuracy","requests":508,"answered":506,"unsupported":2,"errors":0,"median_ms":null},"4":{"score":0.1429,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":12.3},"5":{"score":0.0319,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":17.2},"6":{"score":0.5714,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":26.5},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4241,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":11.4},"11":{"score":0.2901,"jev":0.7175,"metric":"macro-F1","requests":123,"answered":122,"unsupported":1,"errors":0,"median_ms":null},"12":{"score":0.4866,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":4.6},"20":{"score":0.5158,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":4.6},"21":{"score":0.4722,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":4.2},"22":{"score":0.051,"jev":0.1811,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":259.9},"23":{"score":0.5234,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":5.8},"24":{"score":0.3068,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":4.5},"25":{"score":0.2755,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.4697,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":4.3},"27":{"score":0.2858,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":4.4},"28":{"score":0.5051,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":4.1},"29":{"score":0.3312,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":5.2},"30":{"score":0.2165,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":5.0},"31":{"score":0.0768,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":10.2},"32":{"score":0.4322,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.003,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":276.8},"36":{"score":0.199,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.244,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":5.8},"38":{"score":0.0345,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":77.9},"39":{"score":0.6098,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.149,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":4.2,"tracks":[{"track":"A · Arabic","score":0.0184,"headline":false},{"track":"A · English","score":0.149,"headline":true},{"track":"C · Arabic pairs","score":0.535,"headline":false},{"track":"C · English pairs","score":0.41,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4053,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":4.8},"42":{"score":0.4772,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":6.5},"43":{"score":0.4018,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5288,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":4.5},"45":{"score":0.14,"jev":0.204,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":null},"48":{"score":0.411,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":8.1},"50":{"score":0.3341,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":9.6},"56":{"score":0.5015,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":26.4,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.1361,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":5.0,"edition_added":"v0.2","jev":0.827},"58":{"score":0.341,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":4.7,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.1194,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":8.9,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.2708,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.4882,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":8.6,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5575,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":6.4,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":27.53,"balanced_skill":6.04,"breadth_skill":5.89},"frozen_scores":{"balanced_raw":24.71,"balanced_skill":3.47,"breadth_skill":3.31},"categories":[{"id":"knowledge","raw":0.2528,"skill":0.0356,"coverage":0.9998,"pending":0,"n":10},{"id":"language","raw":0.381,"skill":0.0903,"coverage":0.9992,"pending":0,"n":10},{"id":"retrieval","raw":0.2654,"skill":0.0715,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.1557,"skill":0.0582,"coverage":0.9978,"pending":0,"n":5},{"id":"arts","raw":0.2983,"skill":0.0289,"coverage":0.9999,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.3813,"skill":0.1648,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.1269,"skill":0.0,"coverage":0.9927,"random":0.1341,"tracks":[]},"6":{"raw":0.5714,"skill":0.0397,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.5623,"headline":false},{"track":"RouterBench-5shot","score":0.5805,"headline":false}]},"11":{"raw":0.2875,"skill":0.0,"coverage":0.9919,"random":0.3085,"tracks":[]},"20":{"raw":0.5136,"skill":0.0271,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.4722,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0472,"skill":0.0398,"coverage":0.9995,"random":0.0078,"tracks":[]},"23":{"raw":0.5095,"skill":0.019,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3091,"skill":0.0788,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2755,"skill":0.034,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.2165,"skill":0.0506,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.2911,"headline":false},{"track":"GSM8K-10choice","score":0.1418,"headline":false}]},"31":{"raw":0.0768,"skill":0.0,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.199,"skill":0.0938,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.244,"skill":0.0517,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.149,"skill":0.0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0184,"headline":false},{"track":"A · English","score":0.149,"headline":true},{"track":"C · Arabic pairs","score":0.535,"headline":false},{"track":"C · English pairs","score":0.41,"headline":false}]},"41":{"raw":0.4053,"skill":0.108,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4018,"skill":0.0508,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5288,"skill":0.0576,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3341,"skill":0.0336,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5015,"skill":0.003,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.1361,"skill":0.0283,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.341,"skill":0.0448,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.1194,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.2708,"skill":0.0885,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.4882,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5575,"skill":0.115,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.4322,"skill":0.0973,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.003,"skill":0.0,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1397,"skill":0.0,"coverage":0.998,"random":0.1641,"tracks":[]},"12":{"raw":0.4866,"skill":0.231,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5051,"skill":0.0102,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.3312,"skill":0.1083,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0345,"skill":0.0036,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.6098,"skill":0.4261,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.4772,"skill":0.0,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.1429,"skill":0.1319,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.0319,"skill":0.0261,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.1141,"skill":0.0971,"coverage":0.9961,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.7594,"pending":0.24,"latency_rows":749,"latency_path":"laya Agent.accelerate(use_graphs=True): TileLang fused kernels + CUDA graphs per shape bucket, built with the CUDA 13.0 nvcc from pip (nvidia-cuda-nvcc 13.0) because the card image ships nvcc 12.3 without sm_120a; measured with a warm TileLang JIT cache (a first pass with a cold cache had 72 rows of ~5.3 s kernel compilation, kept as latency-v1-full.fast-coldjit); p95 still includes first-use CUDA graph capture per bucket. Stock path (19.8 / 51.2 ms) kept as latency-v1-full.stock"},{"engine":"lfm2600","name":"LFM2.5-2.6B-RLCD","long_name":"LFM2.5-2.6B-RLCD","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":2697198592,"params_estimated":false,"params_from_name":false,"trained_bytes":5394427456,"model_url":"https://huggingface.co/monotykamary/LFM2.5-2.6B-RLCD","code_url":null,"base_model":"LiquidAI/LFM2.5-2.6B-Base","base_url":"https://huggingface.co/LiquidAI/LFM2.5-2.6B-Base","weights_repo":"monotykamary/LFM2.5-2.6B-RLCD","served_checkpoint":null},"completed":120340,"counts":{"ok":119807,"unsupported":533},"lifted":true,"calibration":{"acc":0.3937,"conf":0.6482,"ece":0.2545,"brier":0.7921,"over95":0.0508,"n":72594,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0001,"conf":0.0821,"acc":0.0},{"lo":0.1,"w":0.0102,"conf":0.1744,"acc":0.0494},{"lo":0.2,"w":0.0546,"conf":0.2555,"acc":0.0885},{"lo":0.3,"w":0.0881,"conf":0.3505,"acc":0.1509},{"lo":0.4,"w":0.1035,"conf":0.4478,"acc":0.2711},{"lo":0.5,"w":0.1837,"conf":0.5506,"acc":0.4199},{"lo":0.6,"w":0.136,"conf":0.6498,"acc":0.4539},{"lo":0.7,"w":0.1393,"conf":0.7478,"acc":0.4475},{"lo":0.8,"w":0.1117,"conf":0.8491,"acc":0.4569},{"lo":0.9,"w":0.1729,"conf":0.9643,"acc":0.5483}],"areas":{"knowledge":{"ece":0.181,"acc":0.4212,"conf":0.6022},"language":{"ece":0.2881,"acc":0.4095,"conf":0.6976},"retrieval":{"ece":0.2867,"acc":0.3718,"conf":0.6584},"tools":{"ece":0.5343,"acc":0.2426,"conf":0.7769},"arts":{"ece":0.1619,"acc":0.413,"conf":0.5749}}},"gaps":{"share":0.0044,"reasons":[{"reason":"request exceeds conservative GPU memory budget; reduce prompt or branch batch size","share":0.0044}]},"latency":{"median":39.3,"p95":95.4,"mean":54.2},"results":{"1":{"score":0.3872,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":46.5},"2":{"score":0.1217,"jev":0.6579,"metric":"nDCG@10","requests":685,"answered":640,"unsupported":45,"errors":0,"median_ms":null},"3":{"score":0.0177,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.1778,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":67.0},"5":{"score":0.0732,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":95.3},"6":{"score":0.5927,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":70.7},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5294,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":51.5},"11":{"score":0.3112,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.3486,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":38.5},"20":{"score":0.6016,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":38.5},"21":{"score":0.4981,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":38.4},"22":{"score":0,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":165.1},"23":{"score":0.512,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":38.7},"24":{"score":0.4916,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":38.6},"25":{"score":0.352,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.8131,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":38.4},"27":{"score":0.6596,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":38.6},"28":{"score":0.5083,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":38.2},"29":{"score":0.4556,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":39.2},"30":{"score":0.2051,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":39.1},"31":{"score":0.1048,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":56.0},"32":{"score":0.4707,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0006,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":54.7},"36":{"score":0.1825,"jev":0.4341,"metric":"nDCG@10","requests":220,"answered":128,"unsupported":92,"errors":0,"median_ms":null},"37":{"score":0.2226,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39.0},"38":{"score":0.0561,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":422.6},"39":{"score":0.3685,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2629,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":38.2,"tracks":[{"track":"A · Arabic","score":0.2773,"headline":false},{"track":"A · English","score":0.2629,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.55,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3628,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":38.4},"42":{"score":0.3342,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":41.3},"43":{"score":0.3684,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5158,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":38.3},"45":{"score":0.1357,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.3034,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":43.1},"50":{"score":0.4075,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":48.8},"56":{"score":0.6695,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":50.9,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.2461,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":39.5,"edition_added":"v0.2","jev":0.827},"58":{"score":0.4265,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":38.7,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.2738,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":44.9,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.2311,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.5177,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":44.9,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5248,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":38.8,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":28.22,"balanced_skill":6.76,"breadth_skill":6.64},"frozen_scores":{"balanced_raw":26.19,"balanced_skill":5.0,"breadth_skill":4.62},"categories":[{"id":"knowledge","raw":0.2831,"skill":0.0742,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.3574,"skill":0.0542,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.2796,"skill":0.1053,"coverage":0.924,"pending":0,"n":6},{"id":"tools","raw":0.1617,"skill":0.0443,"coverage":0.9878,"pending":0,"n":5},{"id":"arts","raw":0.3106,"skill":0.0525,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.3872,"skill":0.1728,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.1137,"skill":0.0,"coverage":0.9343,"random":0.1341,"tracks":[]},"6":{"raw":0.5927,"skill":0.0442,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.5491,"headline":false},{"track":"RouterBench-5shot","score":0.6363,"headline":false}]},"11":{"raw":0.3112,"skill":0.0039,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.5994,"skill":0.1988,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.4981,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0,"skill":0.0,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.4999,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5022,"skill":0.3363,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.352,"skill":0.1361,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.2051,"skill":0.0375,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.2866,"headline":false},{"track":"GSM8K-10choice","score":0.1236,"headline":false}]},"31":{"raw":0.1048,"skill":0.025,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.1062,"skill":0.0,"coverage":0.5818,"random":0.116,"tracks":[]},"37":{"raw":0.2226,"skill":0.0249,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.2629,"skill":0.0518,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2773,"headline":false},{"track":"A · English","score":0.2629,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.55,"headline":false}]},"41":{"raw":0.3628,"skill":0.0442,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3684,"skill":0.0,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5158,"skill":0.0316,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4075,"skill":0.1401,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.6695,"skill":0.339,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.2461,"skill":0.1521,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.4265,"skill":0.1687,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.2738,"skill":0.0317,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.2311,"skill":0.0389,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.5177,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5248,"skill":0.0496,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.4707,"skill":0.1585,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0006,"skill":0.0,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1357,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.3486,"skill":0.0243,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5083,"skill":0.0166,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.4556,"skill":0.2741,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0561,"skill":0.0259,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.3685,"skill":0.0712,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.3342,"skill":0.0,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.1778,"skill":0.1672,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.0732,"skill":0.0676,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.0177,"skill":0.0,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.7406,"pending":0.24,"latency_rows":748,"latency_path":"author pcd Engine.constrained, board guard lifts, fleet venv torch 2.14.0+cu130"},{"engine":"lfm350-sdpa","name":"LFM2.5-350M-RLCD","long_name":"LFM2.5-350M-RLCD","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":354483968,"params_estimated":false,"params_from_name":false,"trained_bytes":708984464,"model_url":"https://huggingface.co/notnotsamuel/LFM2.5-350M-RLCD","code_url":null,"base_model":null,"base_url":null,"weights_repo":"notnotsamuel/LFM2.5-350M-RLCD","served_checkpoint":null},"completed":120340,"counts":{"ok":120340},"lifted":false,"calibration":{"acc":0.2785,"conf":0.8464,"ece":0.5681,"brier":1.2239,"over95":0.3378,"n":72594,"benchmarks":32,"rel":[{"lo":0.0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0003,"conf":0.192,"acc":0.5766},{"lo":0.2,"w":0.0075,"conf":0.262,"acc":0.0486},{"lo":0.3,"w":0.0186,"conf":0.3519,"acc":0.0377},{"lo":0.4,"w":0.0316,"conf":0.4506,"acc":0.0665},{"lo":0.5,"w":0.0821,"conf":0.5496,"acc":0.2987},{"lo":0.6,"w":0.0851,"conf":0.6515,"acc":0.3012},{"lo":0.7,"w":0.0945,"conf":0.7502,"acc":0.2712},{"lo":0.8,"w":0.119,"conf":0.8504,"acc":0.2794},{"lo":0.9,"w":0.5614,"conf":0.9815,"acc":0.2961}],"areas":{"knowledge":{"ece":0.5573,"acc":0.2783,"conf":0.835},"language":{"ece":0.5795,"acc":0.279,"conf":0.8586},"retrieval":{"ece":0.6173,"acc":0.1868,"conf":0.8041},"tools":{"ece":0.5431,"acc":0.2504,"conf":0.7935},"arts":{"ece":0.5467,"acc":0.3684,"conf":0.9152}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":26.7,"p95":174.3,"mean":46.5},"results":{"1":{"score":0.3837,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":27.9},"2":{"score":0.1233,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":328.8},"3":{"score":0.0236,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.0018,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":91.6},"5":{"score":0.0029,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":174.3},"6":{"score":0.5035,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":49.1},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.3421,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":28.3},"11":{"score":0.2021,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.1962,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":25.0},"20":{"score":0.5848,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":25.3},"21":{"score":0.4958,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":24.7},"22":{"score":0.051,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":267.6},"23":{"score":0.4814,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":26.0},"24":{"score":0.2389,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":25.6},"25":{"score":0.2704,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.2647,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":25.2},"27":{"score":0.2304,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":25.8},"28":{"score":0.5201,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":25.0},"29":{"score":0.2515,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":26.3},"30":{"score":0.1827,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":25.9},"31":{"score":0.0796,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":47.1},"32":{"score":0.3617,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0133,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":35.7},"36":{"score":0.1103,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.1238,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":26.4},"38":{"score":0.031,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":388.8},"39":{"score":0.3315,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2495,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":24.9,"tracks":[{"track":"A · Arabic","score":0.2505,"headline":false},{"track":"A · English","score":0.2495,"headline":true},{"track":"C · Arabic pairs","score":0.51,"headline":false},{"track":"C · English pairs","score":0.455,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.2532,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":25.4},"42":{"score":0.4288,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":26.9},"43":{"score":0.3228,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.4974,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":24.9},"45":{"score":0.1357,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.6215,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":27.2},"50":{"score":0.3037,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":30.1},"56":{"score":0.5,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":46.0,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.1169,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":26.6,"edition_added":"v0.2","jev":0.827},"58":{"score":0.3018,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":25.4,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.1525,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":28.0,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.1875,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.5187,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":27.4,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5008,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":26.7,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":22.92,"balanced_skill":1.38,"breadth_skill":1.3},"frozen_scores":{"balanced_raw":22.25,"balanced_skill":1.77,"breadth_skill":1.66},"categories":[{"id":"knowledge","raw":0.2266,"skill":0.0047,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.2996,"skill":0.0098,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.1913,"skill":0.0002,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.1416,"skill":0.0384,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.2903,"skill":0.0298,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.3837,"skill":0.168,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.1233,"skill":0.0,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.5036,"skill":0.0,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.41,"headline":false},{"track":"RouterBench-5shot","score":0.5972,"headline":false}]},"11":{"raw":0.2021,"skill":0.0,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.5865,"skill":0.1731,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.4958,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0485,"skill":0.0411,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.4679,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.2408,"skill":0.0,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2704,"skill":0.0272,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.1827,"skill":0.0096,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.2593,"headline":false},{"track":"GSM8K-10choice","score":0.1061,"headline":false}]},"31":{"raw":0.0796,"skill":0.0,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.1103,"skill":0.0,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.1238,"skill":0.0,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.2495,"skill":0.0345,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2505,"headline":false},{"track":"A · English","score":0.2495,"headline":true},{"track":"C · Arabic pairs","score":0.51,"headline":false},{"track":"C · English pairs","score":0.455,"headline":false}]},"41":{"raw":0.2532,"skill":0.0,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3228,"skill":0.0,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.4974,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3037,"skill":0.0,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.1169,"skill":0.0067,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.3018,"skill":0.0,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.1525,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.1875,"skill":0.0,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.5187,"skill":0.0021,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5008,"skill":0.0016,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.3617,"skill":0.0,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0133,"skill":0.0002,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1357,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.1962,"skill":0.0,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5201,"skill":0.0402,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.2515,"skill":0.002,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.031,"skill":0.0,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.3315,"skill":0.0168,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.4288,"skill":0.0,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.0018,"skill":0.0,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.0029,"skill":0.0,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.0236,"skill":0.0048,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author rlcd Engine with SDPA attention (board path), fleet venv torch 2.14.0+cu130"},{"engine":"lumma-fev-0.1b","name":"Lumma-Fev-0.1B","long_name":"Lumma-Fev-0.1B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":153412928,"params_estimated":false,"params_from_name":false,"trained_bytes":309928008,"model_url":"https://huggingface.co/FrontiersMind/Lumma-fev-0.1b","code_url":null,"base_model":"FrontiersMind/Nandi-Mini-150M","base_url":"https://huggingface.co/FrontiersMind/Nandi-Mini-150M","weights_repo":"FrontiersMind/Lumma-fev-0.1b","served_checkpoint":null},"completed":120340,"counts":{"ok":117795,"unsupported":2093},"lifted":false,"calibration":{"acc":0.3653,"conf":0.5227,"ece":0.1574,"brier":0.7407,"over95":0.0029,"n":34763,"benchmarks":31,"rel":[{"lo":0.0,"w":0.0374,"conf":0.0794,"acc":0.0565},{"lo":0.1,"w":0.073,"conf":0.1357,"acc":0.097},{"lo":0.2,"w":0.0329,"conf":0.2606,"acc":0.1588},{"lo":0.3,"w":0.1038,"conf":0.3572,"acc":0.2015},{"lo":0.4,"w":0.147,"conf":0.4502,"acc":0.2854},{"lo":0.5,"w":0.2484,"conf":0.5457,"acc":0.4538},{"lo":0.6,"w":0.158,"conf":0.6465,"acc":0.4805},{"lo":0.7,"w":0.1074,"conf":0.7464,"acc":0.4738},{"lo":0.8,"w":0.0662,"conf":0.8472,"acc":0.5215},{"lo":0.9,"w":0.0259,"conf":0.931,"acc":0.5406}],"areas":{"knowledge":{"ece":0.19,"acc":0.3139,"conf":0.504},"language":{"ece":0.1303,"acc":0.4631,"conf":0.5931},"retrieval":{"ece":0.13,"acc":0.2085,"conf":0.332},"tools":{"ece":0.0949,"acc":0.3701,"conf":0.4599},"arts":{"ece":0.214,"acc":0.4399,"conf":0.6538}}},"gaps":{"share":0.0175,"reasons":[{"reason":"question longer than its 2,048-token context window","share":0.0175}]},"latency":{"median":20.9,"p95":112.5,"mean":34.6},"results":{"1":{"score":0.0124,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":29.3},"2":{"score":0.1306,"jev":0.6604,"metric":"nDCG@10","requests":685,"answered":632,"unsupported":53,"errors":0,"median_ms":null},"3":{"score":0.0827,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.0002,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":21.7,"wrong_rows":3077,"contamination":"3,077 BANKING77 test items found in FrontiersMind's own decision datasets count as wrong"},"5":{"score":0.1209,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":22.5},"6":{"score":0.5321,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":112.3},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.3584,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":21.9},"11":{"score":0.2755,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.2668,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":20.0},"20":{"score":0.4912,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20.0},"21":{"score":0.4791,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":19.8},"22":{"score":null,"jev":null,"metric":null,"requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"23":{"score":0.4966,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":21.1},"24":{"score":0.2634,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":19.9},"25":{"score":0.2,"jev":0.7897,"metric":"accuracy","requests":196,"answered":195,"unsupported":1,"errors":0,"median_ms":null},"26":{"score":0.3716,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":19.8},"27":{"score":0.2227,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":19.9},"28":{"score":0.5083,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":19.7},"29":{"score":0.2743,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":20.4},"30":{"score":0.2161,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":20.1},"31":{"score":0.0832,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":21.2},"32":{"score":0.383,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0006,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":69.2},"36":{"score":0.1478,"jev":0.4717,"metric":"nDCG@10","requests":220,"answered":190,"unsupported":30,"errors":0,"median_ms":null},"37":{"score":0.1143,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20.8},"38":{"score":0.0106,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":64.4},"39":{"score":0.2273,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.0502,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":19.9,"tracks":[{"track":"A · Arabic","score":0.0,"headline":false},{"track":"A · English","score":0.0502,"headline":true},{"track":"C · Arabic pairs","score":0.51,"headline":false},{"track":"C · English pairs","score":0.415,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.2727,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":20.2},"42":{"score":0.2903,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":21.2},"43":{"score":0.4158,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5076,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20.0},"45":{"score":0.128,"jev":0.2053,"metric":"accuracy","requests":501,"answered":492,"unsupported":9,"errors":0,"median_ms":null},"48":{"score":0.2402,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":21.4},"50":{"score":0.3061,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":21.5},"56":{"score":0.5015,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":30.6,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.0959,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":20.3,"edition_added":"v0.2","jev":0.827},"58":{"score":0.3076,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":20.2,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.2987,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":21.6,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.3106,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.5209,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":21.4,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5022,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":21.2,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":22.07,"balanced_skill":1.78,"breadth_skill":1.75},"frozen_scores":{"balanced_raw":20.56,"balanced_skill":0.7,"breadth_skill":0.64},"categories":[{"id":"knowledge","raw":0.2296,"skill":0.0151,"coverage":0.9974,"pending":0,"n":10},{"id":"language","raw":0.2742,"skill":0.0062,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.2146,"skill":0.0245,"coverage":0.9752,"pending":0,"n":6},{"id":"tools","raw":0.0988,"skill":0.0265,"coverage":0.9857,"pending":0,"n":5},{"id":"arts","raw":0.2941,"skill":0.0257,"coverage":0.8611,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.0124,"skill":0.0,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.1205,"skill":0.0,"coverage":0.9226,"random":0.1341,"tracks":[]},"6":{"raw":0.5321,"skill":0.0,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.4931,"headline":false},{"track":"RouterBench-5shot","score":0.5711,"headline":false}]},"11":{"raw":0.2755,"skill":0.0,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.4882,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.4791,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0,"skill":0.0,"coverage":0,"random":0.0078,"tracks":[]},"23":{"raw":0.4866,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.2605,"skill":0.0139,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.199,"skill":0.0,"coverage":0.9949,"random":0.25,"tracks":[]},"30":{"raw":0.2161,"skill":0.0537,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.323,"headline":false},{"track":"GSM8K-10choice","score":0.1092,"headline":false}]},"31":{"raw":0.0832,"skill":0.0015,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.1276,"skill":0.0131,"coverage":0.8636,"random":0.116,"tracks":[]},"37":{"raw":0.1143,"skill":0.0,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.0502,"skill":0.0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0,"headline":false},{"track":"A · English","score":0.0502,"headline":true},{"track":"C · Arabic pairs","score":0.51,"headline":false},{"track":"C · English pairs","score":0.415,"headline":false}]},"41":{"raw":0.2727,"skill":0.0,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4158,"skill":0.0731,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5076,"skill":0.0152,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3061,"skill":0.0,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5015,"skill":0.003,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.0959,"skill":0.0,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.3076,"skill":0.0,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.2987,"skill":0.0649,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.3106,"skill":0.1382,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.5209,"skill":0.0066,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5022,"skill":0.0044,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.383,"skill":0.0191,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0006,"skill":0.0,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1257,"skill":0.0,"coverage":0.982,"random":0.1641,"tracks":[]},"12":{"raw":0.2668,"skill":0.0,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5083,"skill":0.0166,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.2743,"skill":0.0324,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0106,"skill":0.0,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.2273,"skill":0.0,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.2903,"skill":0.0,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.0002,"skill":0.0,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.1209,"skill":0.1156,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.0827,"skill":0.065,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0392,"skill":0.0392,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.7112,"pending":0.24,"latency_rows":739,"latency_path":"author model.decide, all questions in one forward, bf16, sdpa backbone (outer attention eager as shipped), torch 2.14.0+cu130","contamination":[{"catalog_id":4,"benchmark":"BANKING77","wrong":3077,"flag":"3,077 BANKING77 test items found in FrontiersMind's own decision datasets count as wrong"}],"new_in_edition":true},{"engine":"lumma-fev-0.6b","name":"Lumma-Fev-0.6B","long_name":"Lumma-Fev-0.6B","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":649281964,"params_estimated":false,"params_from_name":false,"trained_bytes":1300071072,"model_url":"https://huggingface.co/FrontiersMind/Lumma-fev-0.6b","code_url":null,"base_model":"FrontiersMind/Lumma-0.6B-Base","base_url":"https://huggingface.co/FrontiersMind/Lumma-0.6B-Base","weights_repo":"FrontiersMind/Lumma-fev-0.6b","served_checkpoint":null},"completed":120340,"counts":{"ok":119884,"unsupported":4},"lifted":false,"calibration":{"acc":0.3741,"conf":0.6098,"ece":0.2357,"brier":0.8096,"over95":0.0562,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0649,"conf":0.0577,"acc":0.0217},{"lo":0.1,"w":0.0344,"conf":0.139,"acc":0.0954},{"lo":0.2,"w":0.0249,"conf":0.252,"acc":0.1879},{"lo":0.3,"w":0.0696,"conf":0.3589,"acc":0.2749},{"lo":0.4,"w":0.1227,"conf":0.4518,"acc":0.3012},{"lo":0.5,"w":0.1668,"conf":0.5486,"acc":0.4053},{"lo":0.6,"w":0.1348,"conf":0.6485,"acc":0.4017},{"lo":0.7,"w":0.0968,"conf":0.7481,"acc":0.4402},{"lo":0.8,"w":0.1008,"conf":0.8516,"acc":0.4409},{"lo":0.9,"w":0.1844,"conf":0.9622,"acc":0.5415}],"areas":{"knowledge":{"ece":0.2505,"acc":0.3145,"conf":0.565},"language":{"ece":0.1581,"acc":0.5222,"conf":0.6801},"retrieval":{"ece":0.3964,"acc":0.2137,"conf":0.6017},"tools":{"ece":0.1237,"acc":0.3985,"conf":0.5193},"arts":{"ece":0.2902,"acc":0.3526,"conf":0.6427}}},"gaps":{"share":0.0,"reasons":[{"reason":"question longer than its 12,288-token context window","share":0.0}]},"latency":{"median":21.3,"p95":219.7,"mean":48.5},"results":{"1":{"score":0.0561,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":29.3},"2":{"score":0.1529,"jev":0.6532,"metric":"nDCG@10","requests":685,"answered":683,"unsupported":2,"errors":0,"median_ms":404.1},"3":{"score":0.0413,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.0005,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":21.9,"wrong_rows":3077,"contamination":"3,077 BANKING77 test items found in FrontiersMind's own decision datasets count as wrong"},"5":{"score":0.2452,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":22.9},"6":{"score":0.4555,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":221.5},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.0821,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":22.5},"11":{"score":0.2095,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.3049,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":20.7},"20":{"score":0.4852,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20.8},"21":{"score":0.4928,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":20.4},"22":{"score":0.013,"jev":0.1811,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":71.5},"23":{"score":0.487,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":21.5},"24":{"score":0.2855,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":20.5},"25":{"score":0.2041,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.4205,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":20.5},"27":{"score":0.2739,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":20.4},"28":{"score":0.4988,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":20.6},"29":{"score":0.3056,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":21.2},"30":{"score":0.224,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":21.0},"31":{"score":0.0752,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":21.6},"32":{"score":0.3896,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0055,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":82.0},"36":{"score":0.1461,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.2452,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":21.5},"38":{"score":0.0,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":92.5},"39":{"score":0.2533,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.0639,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":21.7,"tracks":[{"track":"A · Arabic","score":0.0,"headline":false},{"track":"A · English","score":0.0639,"headline":true},{"track":"C · Arabic pairs","score":0.47,"headline":false},{"track":"C · English pairs","score":0.47,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3374,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":20.6},"42":{"score":0.4869,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":21.4},"43":{"score":0.4719,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.4986,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20.7},"45":{"score":0.134,"jev":0.204,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":null},"48":{"score":0.2551,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":21.7},"50":{"score":0.3049,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":22.1},"56":{"score":0.5985,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":31.3,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.1103,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":21.0,"edition_added":"v0.2","jev":0.827},"58":{"score":0.3112,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":20.8,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.2588,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":22.1,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.2784,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.4694,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":21.8,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.503,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":21.5,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":24.02,"balanced_skill":2.97,"breadth_skill":2.69},"frozen_scores":{"balanced_raw":21.61,"balanced_skill":1.55,"breadth_skill":1.41},"categories":[{"id":"knowledge","raw":0.2385,"skill":0.0234,"coverage":0.9998,"pending":0,"n":10},{"id":"language","raw":0.2973,"skill":0.0091,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.2753,"skill":0.0888,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.0978,"skill":0.0112,"coverage":0.9995,"pending":0,"n":5},{"id":"arts","raw":0.2873,"skill":0.0144,"coverage":0.9999,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.0561,"skill":0.0,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.1524,"skill":0.0212,"coverage":0.9971,"random":0.1341,"tracks":[]},"6":{"raw":0.4555,"skill":0.0,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.3314,"headline":false},{"track":"RouterBench-5shot","score":0.5797,"headline":false}]},"11":{"raw":0.2095,"skill":0.0,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.4809,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.4928,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0134,"skill":0.0057,"coverage":0.9995,"random":0.0078,"tracks":[]},"23":{"raw":0.4978,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.2847,"skill":0.0463,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2041,"skill":0.0,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.224,"skill":0.0587,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.2881,"headline":false},{"track":"GSM8K-10choice","score":0.16,"headline":false}]},"31":{"raw":0.0752,"skill":0.0,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.1461,"skill":0.0341,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.2452,"skill":0.0533,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.0639,"skill":0.0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0,"headline":false},{"track":"A · English","score":0.0639,"headline":true},{"track":"C · Arabic pairs","score":0.47,"headline":false},{"track":"C · English pairs","score":0.47,"headline":false}]},"41":{"raw":0.3374,"skill":0.0061,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4719,"skill":0.1622,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.4986,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3049,"skill":0.0,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5985,"skill":0.197,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.1103,"skill":0.0,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.3112,"skill":0.0016,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.2588,"skill":0.0117,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.2784,"skill":0.098,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.4694,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.503,"skill":0.006,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.3896,"skill":0.0296,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0055,"skill":0.0,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1337,"skill":0.0,"coverage":0.998,"random":0.1641,"tracks":[]},"12":{"raw":0.3049,"skill":0.0,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.4988,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.3056,"skill":0.0741,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.2533,"skill":0.0,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.4869,"skill":0.0018,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.0005,"skill":0.0,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.2452,"skill":0.2406,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.0413,"skill":0.0228,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.7599,"pending":0.24,"latency_rows":750,"latency_path":"author model.decide, all questions in one forward, bf16, sdpa backbone (outer attention eager as shipped), torch 2.14.0+cu130","contamination":[{"catalog_id":4,"benchmark":"BANKING77","wrong":3077,"flag":"3,077 BANKING77 test items found in FrontiersMind's own decision datasets count as wrong"}],"new_in_edition":true},{"engine":"metask-jev-4b","name":"Metask-Jev-4B","long_name":"Metask-Jev-4B","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":9078620536,"model_url":"https://huggingface.co/wayfind/metask-jev-4b-policy-mix","code_url":"https://github.com/metask-ai/metask-jev","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"wayfind/metask-jev-4b-policy-mix","served_checkpoint":null},"completed":120340,"counts":{"ok":105840,"unsupported":14500},"lifted":false,"calibration":{"acc":0.6241,"conf":0.7244,"ece":0.1035,"brier":0.5026,"over95":0.004,"n":32698,"benchmarks":28,"rel":[{"lo":0.0,"w":0.0101,"conf":0.0831,"acc":0.093},{"lo":0.1,"w":0.0252,"conf":0.1511,"acc":0.1364},{"lo":0.2,"w":0.0293,"conf":0.2593,"acc":0.309},{"lo":0.3,"w":0.048,"conf":0.3518,"acc":0.2295},{"lo":0.4,"w":0.056,"conf":0.4503,"acc":0.3927},{"lo":0.5,"w":0.0939,"conf":0.5523,"acc":0.5137},{"lo":0.6,"w":0.098,"conf":0.6512,"acc":0.5359},{"lo":0.7,"w":0.1444,"conf":0.7533,"acc":0.5933},{"lo":0.8,"w":0.228,"conf":0.8581,"acc":0.6833},{"lo":0.9,"w":0.2669,"conf":0.9362,"acc":0.8817}],"areas":{"knowledge":{"ece":0.091,"acc":0.5421,"conf":0.6256},"language":{"ece":0.1375,"acc":0.6844,"conf":0.8195},"retrieval":{"ece":0.1787,"acc":0.5892,"conf":0.7643},"tools":{"ece":0.0495,"acc":0.9139,"conf":0.9008},"arts":{"ece":0.0894,"acc":0.5862,"conf":0.6742}}},"gaps":{"share":0.1205,"reasons":[{"reason":"metask serve.py HTTP 500: decision: enum choices must be 1–26 nonempty strings.","share":0.1205}]},"latency":{"median":57.3,"p95":369.4,"mean":146.6},"results":{"1":{"score":0.8666,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":116.4},"2":{"score":0.6568,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1778.8},"3":{"score":null,"jev":null,"metric":null,"requests":508,"answered":0,"unsupported":508,"errors":0,"median_ms":null},"4":{"score":null,"jev":null,"metric":null,"requests":3080,"answered":0,"unsupported":3080,"errors":0,"median_ms":null},"5":{"score":null,"jev":null,"metric":null,"requests":5500,"answered":0,"unsupported":5500,"errors":0,"median_ms":null},"6":{"score":0.7962,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":232.7},"9":{"score":0.2045,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5765,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":66.0},"11":{"score":0.6933,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5625,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":55.3},"20":{"score":0.7848,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":54.9},"21":{"score":0.5209,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":53.9},"22":{"score":null,"jev":null,"metric":null,"requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"23":{"score":0.5788,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":56.4},"24":{"score":0.6954,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":55.0},"25":{"score":0.4082,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9541,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":54.5},"27":{"score":0.8993,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":54.7},"28":{"score":0.6709,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":53.5},"29":{"score":0.7528,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":56.5},"30":{"score":0.3446,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":59.2},"31":{"score":0.2305,"jev":0.2746,"metric":"accuracy","requests":5000,"answered":1588,"unsupported":3412,"errors":0,"median_ms":69.0},"32":{"score":0.5479,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.1709,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":445.8},"36":{"score":0.4027,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3488,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":58.9},"38":{"score":0.0675,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":3469.9},"39":{"score":0.793,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4762,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":53.9,"tracks":[{"track":"A · Arabic","score":0.1942,"headline":false},{"track":"A · English","score":0.4762,"headline":true},{"track":"C · Arabic pairs","score":0.565,"headline":false},{"track":"C · English pairs","score":0.805,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4303,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":55.8},"42":{"score":0.7341,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":57.8},"43":{"score":0.4632,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6172,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":54.7},"45":{"score":0.0838,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2699,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":59.5},"50":{"score":0.4004,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":62.9},"56":{"score":0.516,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":493.7,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.4522,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":59.6,"edition_added":"v0.2","jev":0.827},"58":{"score":0.6065,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":56.0,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.4285,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":62.3,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.553,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.6638,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":60.3,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.624,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":56.8,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":42.77,"balanced_skill":26.89,"breadth_skill":25.31},"frozen_scores":{"balanced_raw":38.35,"balanced_skill":22.81,"breadth_skill":19.14},"categories":[{"id":"knowledge","raw":0.3776,"skill":0.2096,"coverage":0.9368,"pending":0,"n":10},{"id":"language","raw":0.5888,"skill":0.3939,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.2988,"skill":0.1291,"coverage":0.6364,"pending":0,"n":6},{"id":"tools","raw":0.4314,"skill":0.3759,"coverage":0.7778,"pending":0,"n":5},{"id":"arts","raw":0.3925,"skill":0.1828,"coverage":0.8611,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.8666,"skill":0.8199,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6568,"skill":0.6036,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7962,"skill":0.5194,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7825,"headline":false},{"track":"RouterBench-5shot","score":0.8099,"headline":false}]},"11":{"raw":0.6933,"skill":0.5565,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.783,"skill":0.566,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5209,"skill":0.0419,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0,"skill":0.0,"coverage":0,"random":0.0078,"tracks":[]},"23":{"raw":0.5688,"skill":0.1376,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7027,"skill":0.6036,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4082,"skill":0.2109,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.3446,"skill":0.211,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.4534,"headline":false},{"track":"GSM8K-10choice","score":0.2358,"headline":false}]},"31":{"raw":0.0732,"skill":0.0,"coverage":0.3176,"random":0.0819,"tracks":[]},"36":{"raw":0.4027,"skill":0.3242,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.3488,"skill":0.1832,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.4762,"skill":0.3261,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1942,"headline":false},{"track":"A · English","score":0.4762,"headline":true},{"track":"C · Arabic pairs","score":0.565,"headline":false},{"track":"C · English pairs","score":0.805,"headline":false}]},"41":{"raw":0.4303,"skill":0.1454,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4632,"skill":0.1482,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6172,"skill":0.2344,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4004,"skill":0.1297,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.516,"skill":0.032,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.4522,"skill":0.3839,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6065,"skill":0.4296,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.4285,"skill":0.238,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.553,"skill":0.4413,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.6638,"skill":0.3029,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.624,"skill":0.248,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5479,"skill":0.2812,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.1709,"skill":0.1599,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.0838,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.5625,"skill":0.3447,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.6709,"skill":0.3418,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.7528,"skill":0.6704,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0675,"skill":0.0377,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.793,"skill":0.6955,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7341,"skill":0.4827,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0127,"tracks":[]},"5":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.006,"tracks":[]},"3":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0189,"tracks":[]},"9":{"raw":0.2045,"skill":0.2045,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.6927,"pending":0.24,"latency_rows":678,"latency_path":"author serve.py /v1/systemone handler in-process (Flask test client), fla kernels; torch 2.14.0+cu130 / transformers 5.17.0 / fla 0.5.2","new_in_edition":true},{"engine":"mini-jev","name":"mini-jev","long_name":"mini-jev Qwen3-4B","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":4022468096,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/r-ms/mini-jev","base_model":"Qwen/Qwen3-4B-Instruct-2507","base_url":"https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507","weights_repo":null,"served_checkpoint":"Qwen/Qwen3-4B-Instruct-2507"},"completed":120340,"counts":{"ok":105840,"unsupported":14500},"lifted":false,"calibration":{"acc":0.604,"conf":0.9497,"ece":0.3457,"brier":0.7257,"over95":0.2734,"n":70199,"benchmarks":28,"rel":[{"lo":0.0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0,"conf":null,"acc":null},{"lo":0.2,"w":0.0008,"conf":0.2683,"acc":0.0},{"lo":0.3,"w":0.0019,"conf":0.3573,"acc":0.081},{"lo":0.4,"w":0.0068,"conf":0.4561,"acc":0.3247},{"lo":0.5,"w":0.0289,"conf":0.5543,"acc":0.3071},{"lo":0.6,"w":0.0302,"conf":0.6506,"acc":0.4033},{"lo":0.7,"w":0.0364,"conf":0.7533,"acc":0.3752},{"lo":0.8,"w":0.0488,"conf":0.8553,"acc":0.3883},{"lo":0.9,"w":0.8463,"conf":0.9936,"acc":0.6475}],"areas":{"knowledge":{"ece":0.3956,"acc":0.5307,"conf":0.9264},"language":{"ece":0.3249,"acc":0.6478,"conf":0.9726},"retrieval":{"ece":0.3419,"acc":0.6439,"conf":0.9857},"tools":{"ece":0.1914,"acc":0.7885,"conf":0.9798},"arts":{"ece":0.3342,"acc":0.5947,"conf":0.9289}}},"gaps":{"share":0.1205,"reasons":[{"reason":"supports only 2–26 answer options per question","share":0.1205},{"reason":"ran out of GPU memory on the longest prompts","share":0.0}]},"latency":{"median":66.9,"p95":178.0,"mean":94.7},"results":{"1":{"score":0.7739,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":72.4},"2":{"score":0.4491,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1117.3},"3":{"score":null,"jev":null,"metric":null,"requests":508,"answered":0,"unsupported":508,"errors":0,"median_ms":null},"4":{"score":null,"jev":null,"metric":null,"requests":3080,"answered":0,"unsupported":3080,"errors":0,"median_ms":null},"5":{"score":null,"jev":null,"metric":null,"requests":5500,"answered":0,"unsupported":5500,"errors":0,"median_ms":null},"6":{"score":0.7938,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":146.6},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5875,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":81.6},"11":{"score":0.5444,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5194,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":65.4},"20":{"score":0.8098,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":65.9},"21":{"score":0.5822,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":65.4},"22":{"score":null,"jev":null,"metric":null,"requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"23":{"score":0.5418,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":66.9},"24":{"score":0.6734,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":65.8},"25":{"score":0.3061,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9579,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":65.8},"27":{"score":0.884,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":65.5},"28":{"score":0.5951,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":65.8},"29":{"score":0.7721,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":66.5},"30":{"score":0.4814,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":65.9},"31":{"score":0.1688,"jev":0.2746,"metric":"accuracy","requests":5000,"answered":1588,"unsupported":3412,"errors":0,"median_ms":67.8},"32":{"score":0.5957,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2079,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":95.0},"36":{"score":0.3296,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.2921,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":67.4},"38":{"score":0.0912,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":359.2},"39":{"score":0.6397,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.3613,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":65.8,"tracks":[{"track":"A · Arabic","score":0.2807,"headline":false},{"track":"A · English","score":0.3613,"headline":true},{"track":"C · Arabic pairs","score":0.75,"headline":false},{"track":"C · English pairs","score":0.865,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4306,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":66.1},"42":{"score":0.7075,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":69.2},"43":{"score":0.3877,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5442,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":66.0},"45":{"score":0.1198,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.4745,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":71.2},"50":{"score":0.4684,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":71.4},"56":{"score":0.5125,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":86.2,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.4169,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":66.6,"edition_added":"v0.2","jev":0.827},"58":{"score":0.5141,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":65.9,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.5027,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":69.5,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.5549,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.0,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":73.4,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5723,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":68.0,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":37.45,"balanced_skill":20.98,"breadth_skill":19.96},"frozen_scores":{"balanced_raw":36.18,"balanced_skill":20.11,"breadth_skill":17.38},"categories":[{"id":"knowledge","raw":0.361,"skill":0.1763,"coverage":0.9368,"pending":0,"n":10},{"id":"language","raw":0.4753,"skill":0.2857,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.2685,"skill":0.0866,"coverage":0.6364,"pending":0,"n":6},{"id":"tools","raw":0.3482,"skill":0.2842,"coverage":0.7778,"pending":0,"n":5},{"id":"arts","raw":0.4092,"skill":0.2107,"coverage":0.8611,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.7739,"skill":0.6948,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.4491,"skill":0.3638,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7938,"skill":0.5141,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7791,"headline":false},{"track":"RouterBench-5shot","score":0.8085,"headline":false}]},"11":{"raw":0.5444,"skill":0.3411,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.8024,"skill":0.6049,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5822,"skill":0.1644,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0,"skill":0.0,"coverage":0,"random":0.0078,"tracks":[]},"23":{"raw":0.5381,"skill":0.0761,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7019,"skill":0.6026,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3061,"skill":0.0748,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.4814,"skill":0.3748,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.5588,"headline":false},{"track":"GSM8K-10choice","score":0.4041,"headline":false}]},"31":{"raw":0.0536,"skill":0.0,"coverage":0.3176,"random":0.0819,"tracks":[]},"36":{"raw":0.3296,"skill":0.2416,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.2921,"skill":0.112,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.3613,"skill":0.1783,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2807,"headline":false},{"track":"A · English","score":0.3613,"headline":true},{"track":"C · Arabic pairs","score":0.75,"headline":false},{"track":"C · English pairs","score":0.865,"headline":false}]},"41":{"raw":0.4306,"skill":0.1459,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3877,"skill":0.0286,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5442,"skill":0.0884,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4684,"skill":0.2284,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5125,"skill":0.025,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.4169,"skill":0.3442,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.5141,"skill":0.2957,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.5027,"skill":0.3369,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.5549,"skill":0.4436,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5723,"skill":0.1446,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5957,"skill":0.3572,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2079,"skill":0.1974,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1198,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.5194,"skill":0.2801,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5951,"skill":0.1902,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.7721,"skill":0.6961,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0912,"skill":0.0621,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.6397,"skill":0.4701,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7075,"skill":0.4309,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0127,"tracks":[]},"5":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.006,"tracks":[]},"3":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.6927,"pending":0.24,"latency_rows":678,"latency_path":"author mini-jev engine in-process, bf16, fleet venv torch 2.14.0+cu130 (same as board)"},{"engine":"mojev","name":"MoJev","long_name":"MoJev Qwen3.5-0.8B","variant":null,"meta":{"kind":"head / adapter","technique":"autoregressive","technique_name":"autoregressive","served_params":873438784,"params_estimated":false,"params_from_name":false,"trained_bytes":1965507264,"model_url":"https://huggingface.co/MoLeMo-Lab/mojev","code_url":"https://github.com/MoLeMo-Lab/mojev","base_model":"Qwen/Qwen3.5-0.8B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B-Base","weights_repo":"MoLeMo-Lab/mojev","served_checkpoint":null},"completed":120340,"counts":{"ok":119898},"lifted":false,"calibration":{"acc":0.4246,"conf":0.4891,"ece":0.1246,"brier":0.7097,"over95":0.0107,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.1468,"conf":0.0384,"acc":0.2167},{"lo":0.1,"w":0.0221,"conf":0.1534,"acc":0.2875},{"lo":0.2,"w":0.0311,"conf":0.2622,"acc":0.2293},{"lo":0.3,"w":0.1171,"conf":0.3556,"acc":0.3627},{"lo":0.4,"w":0.1222,"conf":0.4463,"acc":0.3855},{"lo":0.5,"w":0.2171,"conf":0.5477,"acc":0.5239},{"lo":0.6,"w":0.1458,"conf":0.6463,"acc":0.5669},{"lo":0.7,"w":0.0917,"conf":0.7464,"acc":0.5235},{"lo":0.8,"w":0.0617,"conf":0.8488,"acc":0.462},{"lo":0.9,"w":0.0443,"conf":0.943,"acc":0.3776}],"areas":{"knowledge":{"ece":0.0842,"acc":0.3986,"conf":0.4776},"language":{"ece":0.1538,"acc":0.4506,"conf":0.5896},"retrieval":{"ece":0.22,"acc":0.523,"conf":0.3338},"tools":{"ece":0.272,"acc":0.3693,"conf":0.5089},"arts":{"ece":0.0679,"acc":0.3893,"conf":0.4463}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":46.4,"p95":55.1,"mean":51.7},"results":{"1":{"score":0.5372,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":48.1},"2":{"score":0.4778,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":186.3},"3":{"score":0.2913,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.5187,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":47.5},"5":{"score":0.173,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":48.8},"6":{"score":0.5969,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":52.9},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5946,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":50.0},"11":{"score":0.3317,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.2477,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":45.4},"20":{"score":0.5394,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":45.6},"21":{"score":0.5331,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":45.1},"22":{"score":0.0235,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":101.0},"23":{"score":0.5334,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":46.4},"24":{"score":0.4035,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":45.3},"25":{"score":0.2602,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.7513,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":45.2},"27":{"score":0.5606,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":45.3},"28":{"score":0.5162,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":45.0},"29":{"score":0.3936,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":46.0},"30":{"score":0.2968,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":45.9},"31":{"score":0.0704,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":47.2},"32":{"score":0.4827,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0127,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":49.7},"36":{"score":0.2539,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.2541,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":46.8},"38":{"score":0.032,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":157.1},"39":{"score":0.5949,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2311,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":46.0,"tracks":[{"track":"A · Arabic","score":0.1493,"headline":false},{"track":"A · English","score":0.2311,"headline":true},{"track":"C · Arabic pairs","score":0.42,"headline":false},{"track":"C · English pairs","score":0.41,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4089,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":45.8},"42":{"score":0.5531,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":47.1},"43":{"score":0.3439,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5046,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":45.4},"45":{"score":0.1517,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.3457,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":48.0},"50":{"score":0.3013,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":49.2},"56":{"score":0.5,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":48.0,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.1765,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":45.6,"edition_added":"v0.2","jev":0.827},"58":{"score":0.3966,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":45.9,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.0969,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":48.2,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.4015,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.0158,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":48.0,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.4975,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":47.2,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":31.39,"balanced_skill":11.69,"breadth_skill":10.94},"frozen_scores":{"balanced_raw":27.95,"balanced_skill":7.46,"breadth_skill":6.63},"categories":[{"id":"knowledge","raw":0.2679,"skill":0.0557,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.3355,"skill":0.0908,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.3616,"skill":0.1618,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.2905,"skill":0.2186,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.3244,"skill":0.0672,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.5372,"skill":0.3752,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.4778,"skill":0.3969,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.5969,"skill":0.0914,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.6114,"headline":false},{"track":"RouterBench-5shot","score":0.5824,"headline":false}]},"11":{"raw":0.3317,"skill":0.0335,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.5372,"skill":0.0744,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5331,"skill":0.0662,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0259,"skill":0.0183,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5364,"skill":0.0729,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4139,"skill":0.2186,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2602,"skill":0.0136,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.2968,"skill":0.159,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.4632,"headline":false},{"track":"GSM8K-10choice","score":0.1304,"headline":false}]},"31":{"raw":0.0704,"skill":0.0,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.2539,"skill":0.1559,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.2541,"skill":0.0645,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.2311,"skill":0.0108,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1493,"headline":false},{"track":"A · English","score":0.2311,"headline":true},{"track":"C · Arabic pairs","score":0.42,"headline":false},{"track":"C · English pairs","score":0.41,"headline":false}]},"41":{"raw":0.4089,"skill":0.1134,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3439,"skill":0.0,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5046,"skill":0.0092,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3013,"skill":0.0,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.1765,"skill":0.0738,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.3966,"skill":0.1254,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.0969,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.4015,"skill":0.2519,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.0158,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.4975,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.4827,"skill":0.1776,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0127,"skill":0.0,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1517,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.2477,"skill":0.0,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5162,"skill":0.0324,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.3936,"skill":0.1915,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.032,"skill":0.001,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.5949,"skill":0.4042,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.5531,"skill":0.1305,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.5187,"skill":0.5125,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.173,"skill":0.168,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.2913,"skill":0.2776,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"mojev.serve.Engine.answer, one packed forward over the state and every question (eager, author Space setting); torch 2.14.0+cu130","new_in_edition":true},{"engine":"nimble-v2","name":"Bespoke Nimble 9B v2","long_name":"Bespoke Nimble 9B v2","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":9653104368,"params_estimated":false,"params_from_name":false,"trained_bytes":173188512,"model_url":"https://huggingface.co/bespokelabs/Bespoke-Nimble-9B-v2","code_url":"https://github.com/bespokelabsai/nimble","base_model":"Qwen/Qwen3.5-9B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B-Base","weights_repo":"bespokelabs/Bespoke-Nimble-9B-v2","served_checkpoint":null},"completed":120340,"counts":{"ok":119898},"lifted":false,"calibration":{"acc":0.6425,"conf":0.6479,"ece":0.024,"brier":0.4671,"over95":0.0019,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0464,"conf":0.066,"acc":0.1059},{"lo":0.1,"w":0.0274,"conf":0.1453,"acc":0.2065},{"lo":0.2,"w":0.032,"conf":0.2583,"acc":0.3046},{"lo":0.3,"w":0.0625,"conf":0.3532,"acc":0.3399},{"lo":0.4,"w":0.0698,"conf":0.4535,"acc":0.4014},{"lo":0.5,"w":0.1575,"conf":0.5492,"acc":0.5765},{"lo":0.6,"w":0.1215,"conf":0.6511,"acc":0.6422},{"lo":0.7,"w":0.1332,"conf":0.7523,"acc":0.7196},{"lo":0.8,"w":0.1762,"conf":0.8557,"acc":0.8434},{"lo":0.9,"w":0.1736,"conf":0.9348,"acc":0.9201}],"areas":{"knowledge":{"ece":0.0354,"acc":0.5899,"conf":0.597},"language":{"ece":0.0307,"acc":0.7503,"conf":0.7524},"retrieval":{"ece":0.0992,"acc":0.5737,"conf":0.5898},"tools":{"ece":0.0441,"acc":0.8315,"conf":0.7935},"arts":{"ece":0.0532,"acc":0.506,"conf":0.5309}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":77.3,"p95":621.4,"mean":982.3},"results":{"1":{"score":0.9156,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":169.9},"2":{"score":0.5795,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":31412.8},"3":{"score":0.8051,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.6938,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":143.1},"5":{"score":0.3084,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":254.4},"6":{"score":0.7831,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":431.8},"9":{"score":0.0227,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5992,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":98.0},"11":{"score":0.7032,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6819,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":74.1},"20":{"score":0.8666,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":73.6},"21":{"score":0.5753,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":72.9},"22":{"score":0.1465,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":561.6},"23":{"score":0.543,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":74.7},"24":{"score":0.7724,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":74.3},"25":{"score":0.4388,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9811,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":73.7},"27":{"score":0.9411,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":73.9},"28":{"score":0.7561,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":72.2},"29":{"score":0.8499,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":76.3},"30":{"score":0.5565,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":76.8},"31":{"score":0.1362,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":106.9},"32":{"score":0.5638,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2352,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":713.8},"36":{"score":0.2814,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4553,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":79.2},"38":{"score":0.2314,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":46407.9},"39":{"score":0.6547,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5372,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":74.1,"tracks":[{"track":"A · Arabic","score":0.1293,"headline":false},{"track":"A · English","score":0.5372,"headline":true},{"track":"C · Arabic pairs","score":0.685,"headline":false},{"track":"C · English pairs","score":0.875,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3913,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":74.8},"42":{"score":0.7633,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":76.1},"43":{"score":0.5632,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6692,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":74.0},"45":{"score":0.0858,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2239,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":79.9},"50":{"score":0.4039,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":93.9},"56":{"score":0.656,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":762.1,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.5172,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":79.1,"edition_added":"v0.2","jev":0.827},"58":{"score":0.6744,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":74.9,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.7267,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":85.3,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.5568,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.6507,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":79.4,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.6295,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":77.9,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":53.61,"balanced_skill":39.57,"breadth_skill":38.34},"frozen_scores":{"balanced_raw":41.3,"balanced_skill":27.11,"breadth_skill":23.1},"categories":[{"id":"knowledge","raw":0.4429,"skill":0.2952,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.6299,"skill":0.4509,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.4971,"skill":0.3493,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.6285,"skill":0.5921,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4436,"skill":0.2471,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9156,"skill":0.886,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.5795,"skill":0.5144,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7832,"skill":0.4907,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7612,"headline":false},{"track":"RouterBench-5shot","score":0.8051,"headline":false}]},"11":{"raw":0.7032,"skill":0.5708,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.8624,"skill":0.7248,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5753,"skill":0.1507,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.1363,"skill":0.1295,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5339,"skill":0.0677,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7844,"skill":0.7125,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4388,"skill":0.2517,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.5565,"skill":0.4679,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.6467,"headline":false},{"track":"GSM8K-10choice","score":0.4663,"headline":false}]},"31":{"raw":0.1362,"skill":0.0592,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.2814,"skill":0.1871,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.4553,"skill":0.3168,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.5372,"skill":0.4047,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1293,"headline":false},{"track":"A · English","score":0.5372,"headline":true},{"track":"C · Arabic pairs","score":0.685,"headline":false},{"track":"C · English pairs","score":0.875,"headline":false}]},"41":{"raw":0.3913,"skill":0.087,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.5632,"skill":0.3069,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6692,"skill":0.3384,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4039,"skill":0.1349,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.656,"skill":0.312,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.5172,"skill":0.457,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6744,"skill":0.528,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.7267,"skill":0.6356,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.5568,"skill":0.446,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.6507,"skill":0.2758,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.6295,"skill":0.259,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5638,"skill":0.3065,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2352,"skill":0.225,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.0858,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.6819,"skill":0.5235,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.7561,"skill":0.5122,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.8499,"skill":0.7999,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.2314,"skill":0.2068,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.6547,"skill":0.4921,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7633,"skill":0.5395,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.6938,"skill":0.6899,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.3084,"skill":0.3042,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.8051,"skill":0.8013,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0227,"skill":0.0227,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.1044,"skill":0.1044,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"ParallelScorer.score (one full-prompt forward per question), author pins torch 2.8.0+cu128 / transformers 5.17.0 / fla 0.5.2 / causal-conv1d 1.7.0, Triton autotune cache from the board HF Job"},{"engine":"openvons","name":"openvons","long_name":"openvons Qwen3-4B","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":4022468096,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/genai-craft/openvons","base_model":"Qwen/Qwen3-4B-Instruct-2507","base_url":"https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507","weights_repo":null,"served_checkpoint":"Qwen/Qwen3-4B-Instruct-2507"},"completed":120340,"counts":{"ok":120340},"lifted":false,"calibration":{"acc":0.583,"conf":0.9542,"ece":0.3712,"brier":0.7749,"over95":0.3051,"n":72594,"benchmarks":32,"rel":[{"lo":0.0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0,"conf":null,"acc":null},{"lo":0.2,"w":0.0007,"conf":0.2424,"acc":0.0},{"lo":0.3,"w":0.0026,"conf":0.3383,"acc":0.0431},{"lo":0.4,"w":0.0116,"conf":0.4512,"acc":0.1647},{"lo":0.5,"w":0.0223,"conf":0.5416,"acc":0.3145},{"lo":0.6,"w":0.0276,"conf":0.6423,"acc":0.2856},{"lo":0.7,"w":0.0254,"conf":0.7429,"acc":0.3081},{"lo":0.8,"w":0.0386,"conf":0.8415,"acc":0.2981},{"lo":0.9,"w":0.8712,"conf":0.9949,"acc":0.6276}],"areas":{"knowledge":{"ece":0.3846,"acc":0.5485,"conf":0.933},"language":{"ece":0.3203,"acc":0.6607,"conf":0.981},"retrieval":{"ece":0.4285,"acc":0.54,"conf":0.9685},"tools":{"ece":0.28,"acc":0.7086,"conf":0.9886},"arts":{"ece":0.4422,"acc":0.478,"conf":0.9202}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":21.2,"p95":113.3,"mean":44.1},"results":{"1":{"score":0.8406,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":42.2},"2":{"score":0.6032,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":252.4},"3":{"score":0.435,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.3255,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":78.4},"5":{"score":0.4172,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":107.3},"6":{"score":0.798,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":85.5},"9":{"score":0.0455,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6232,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":46.3},"11":{"score":0.5355,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5379,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":17.6},"20":{"score":0.7622,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":15.7},"21":{"score":0.5906,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":14.1},"22":{"score":0.056,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":294.3},"23":{"score":0.5326,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":24.3},"24":{"score":0.6918,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":15.4},"25":{"score":0.3878,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.963,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":14.7},"27":{"score":0.8951,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":14.6},"28":{"score":0.6259,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":14.4},"29":{"score":0.8076,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":18.2},"30":{"score":0.4587,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":18.3},"31":{"score":0.109,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":71.6},"32":{"score":0.6011,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2612,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":58.9},"36":{"score":0.4048,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.33,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":21.4},"38":{"score":0.1424,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":276.4},"39":{"score":0.6046,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.3333,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":15.0,"tracks":[{"track":"A · Arabic","score":0.3208,"headline":false},{"track":"A · English","score":0.3333,"headline":true},{"track":"C · Arabic pairs","score":0.735,"headline":false},{"track":"C · English pairs","score":0.87,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4249,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":17.9},"42":{"score":0.7,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":28.2},"43":{"score":0.4263,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5578,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":17.2},"45":{"score":0.1138,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.3874,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":30.6},"50":{"score":0.4588,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":37.0},"56":{"score":0.5135,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":54.2,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.4318,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":18.1,"edition_added":"v0.2","jev":0.827},"58":{"score":0.5328,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":17.9,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.5298,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":32.1,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.5871,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.5512,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":32.3,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.6095,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":24.3,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":45.65,"balanced_skill":28.42,"breadth_skill":27.6},"frozen_scores":{"balanced_raw":38.13,"balanced_skill":22.19,"breadth_skill":19.04},"categories":[{"id":"knowledge","raw":0.3864,"skill":0.2077,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.5337,"skill":0.301,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.4288,"skill":0.2537,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.5017,"skill":0.4465,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4108,"skill":0.2033,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.8406,"skill":0.7848,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6032,"skill":0.5418,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.798,"skill":0.5234,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7851,"headline":false},{"track":"RouterBench-5shot","score":0.8108,"headline":false}]},"11":{"raw":0.5355,"skill":0.3282,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.7561,"skill":0.5122,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5906,"skill":0.1811,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.051,"skill":0.0435,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5143,"skill":0.0287,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7191,"skill":0.6255,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3878,"skill":0.1837,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.4587,"skill":0.3503,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.5663,"headline":false},{"track":"GSM8K-10choice","score":0.351,"headline":false}]},"31":{"raw":0.109,"skill":0.0296,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4048,"skill":0.3267,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.33,"skill":0.1597,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.3333,"skill":0.1423,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3208,"headline":false},{"track":"A · English","score":0.3333,"headline":true},{"track":"C · Arabic pairs","score":0.735,"headline":false},{"track":"C · English pairs","score":0.87,"headline":false}]},"41":{"raw":0.4249,"skill":0.1374,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4263,"skill":0.0898,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5578,"skill":0.1156,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4588,"skill":0.2146,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5135,"skill":0.027,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.4318,"skill":0.3609,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.5328,"skill":0.3228,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.5298,"skill":0.3731,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.5871,"skill":0.4839,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.5512,"skill":0.0695,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.6095,"skill":0.219,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.6011,"skill":0.3658,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2612,"skill":0.2514,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1138,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.5379,"skill":0.3078,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.6259,"skill":0.2518,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.8076,"skill":0.7435,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.1424,"skill":0.115,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.6046,"skill":0.4184,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7,"skill":0.4163,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.3255,"skill":0.3168,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.4172,"skill":0.4137,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.435,"skill":0.4241,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0455,"skill":0.0455,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author client over vLLM 0.29.0 OpenAI server, Qwen3-4B-Instruct-2507 bf16, prefix caching, CUDA graphs, gpu util 0.8"},{"engine":"pngwn-space-uncapped-full","name":"open-jev (pngwn)","long_name":"open-jev (pngwn)","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":121956232,"model_url":"https://huggingface.co/pngwn/system-one-qwen3.5-4b-scorer-v2b","code_url":null,"base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"pngwn/system-one-qwen3.5-4b-scorer-v2b","served_checkpoint":null},"completed":120340,"counts":{"ok":120139},"lifted":true,"calibration":{"acc":0.586,"conf":0.5978,"ece":0.0589,"brier":0.5446,"over95":0.0145,"n":72589,"benchmarks":32,"rel":[{"lo":0.0,"w":0.1012,"conf":0.0513,"acc":0.2456},{"lo":0.1,"w":0.0261,"conf":0.1465,"acc":0.294},{"lo":0.2,"w":0.0283,"conf":0.2455,"acc":0.223},{"lo":0.3,"w":0.0438,"conf":0.3567,"acc":0.3134},{"lo":0.4,"w":0.0789,"conf":0.4535,"acc":0.4291},{"lo":0.5,"w":0.1897,"conf":0.5494,"acc":0.5326},{"lo":0.6,"w":0.1517,"conf":0.6477,"acc":0.6145},{"lo":0.7,"w":0.1188,"conf":0.7489,"acc":0.7098},{"lo":0.8,"w":0.1173,"conf":0.8501,"acc":0.8339},{"lo":0.9,"w":0.1442,"conf":0.9663,"acc":0.8544}],"areas":{"knowledge":{"ece":0.1066,"acc":0.5297,"conf":0.6363},"language":{"ece":0.0192,"acc":0.698,"conf":0.707},"retrieval":{"ece":0.2241,"acc":0.5681,"conf":0.3573},"tools":{"ece":0.1496,"acc":0.7304,"conf":0.6205},"arts":{"ece":0.0824,"acc":0.436,"conf":0.4955}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":112.3,"p95":724.1,"mean":312.7},"results":{"1":{"score":0.8973,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":113.1},"2":{"score":0.6052,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":10281.6},"3":{"score":0.4843,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.5048,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":200.7},"5":{"score":0.5762,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":387.0},"6":{"score":0.7988,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":153.2},"9":{"score":0.0341,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5904,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":113.7},"11":{"score":0.6368,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5879,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":111.3},"20":{"score":0.519,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":111.1},"21":{"score":0.5521,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":110.9},"22":{"score":0.056,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":637.4},"23":{"score":0.5828,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":112.4},"24":{"score":0.4449,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":110.7,"wrong_rows":5303,"contamination":"5,308 contaminated MMLU test items (in its training data) count as wrong"},"25":{"score":0.4388,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9621,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":110.2},"27":{"score":0.9121,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":110.3},"28":{"score":0.708,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":110.0},"29":{"score":0.7713,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":111.2},"30":{"score":0.3669,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":111.8},"31":{"score":0.1004,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":747.6},"32":{"score":0.5811,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3073,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":126.3},"36":{"score":0.4057,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3067,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":112.4},"38":{"score":0.1214,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":519.1},"39":{"score":0.7463,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.3756,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":110.5,"tracks":[{"track":"A · Arabic","score":0.3459,"headline":false},{"track":"A · English","score":0.3756,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.675,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3828,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":111.1},"42":{"score":0.7434,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":112.6},"43":{"score":0.4825,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6316,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":110.2},"45":{"score":0.1098,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2185,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":113.5},"50":{"score":0.4206,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":114.4},"56":{"score":0.5025,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":113.8,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.1272,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":111.9,"edition_added":"v0.2","wrong_rows":9542,"contamination":"9,542 contaminated MMLU-Pro test items (in its training data) count as wrong","jev":0.827},"58":{"score":0.5929,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":111.7,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.4663,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":113.3,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.4564,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.0038,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":112.5,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5533,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":112.4,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":45.67,"balanced_skill":29.91,"breadth_skill":28.6},"frozen_scores":{"balanced_raw":38.03,"balanced_skill":21.57,"breadth_skill":17.72},"categories":[{"id":"knowledge","raw":0.3697,"skill":0.1988,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.518,"skill":0.3546,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.4767,"skill":0.2912,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.5118,"skill":0.4573,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.3823,"skill":0.1414,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.8973,"skill":0.8613,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6052,"skill":0.5441,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7989,"skill":0.5257,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7854,"headline":false},{"track":"RouterBench-5shot","score":0.8123,"headline":false}]},"11":{"raw":0.6368,"skill":0.4747,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.5395,"skill":0.079,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5521,"skill":0.1043,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0535,"skill":0.0461,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.579,"skill":0.158,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8357,"skill":0.7809,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4388,"skill":0.2517,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.3669,"skill":0.2372,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.4655,"headline":false},{"track":"GSM8K-10choice","score":0.2684,"headline":false}]},"31":{"raw":0.1004,"skill":0.0202,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4057,"skill":0.3276,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.3067,"skill":0.1304,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.3756,"skill":0.1967,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3459,"headline":false},{"track":"A · English","score":0.3756,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.675,"headline":false}]},"41":{"raw":0.3828,"skill":0.0742,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4825,"skill":0.1789,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6316,"skill":0.2632,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4206,"skill":0.1592,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5025,"skill":0.005,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.1272,"skill":0.0183,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.5929,"skill":0.4099,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.4663,"skill":0.2884,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.4564,"skill":0.3205,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.0038,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5533,"skill":0.1066,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5811,"skill":0.334,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.3073,"skill":0.2981,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1098,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.5879,"skill":0.3827,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.708,"skill":0.416,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.7713,"skill":0.6951,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.1214,"skill":0.0933,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.7463,"skill":0.6269,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7434,"skill":0.5008,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.5048,"skill":0.4984,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.5762,"skill":0.5736,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.4843,"skill":0.4744,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0341,"skill":0.0341,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.126,"skill":0.126,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"Space cached scorer (shared prefix, chunked branch forwards), v2b merged LoRA bf16, sdpa, uncapped guards as board; fleet venv torch 2.14.0+cu130 (causal-conv1d absent: torch conv fallback)","contamination":[{"catalog_id":57,"benchmark":"MMLU-Pro","wrong":9542,"flag":"9,542 contaminated MMLU-Pro test items (in its training data) count as wrong"},{"catalog_id":24,"benchmark":"MMLU","wrong":5308,"flag":"5,308 contaminated MMLU test items (in its training data) count as wrong"}]},{"engine":"razorback-one-read-3e296f08","name":"razorback16 openjev diffusiongemma (NVFP4, vLLM)","long_name":"razorback16 diffgemma","variant":"one read, NVFP4","meta":{"kind":"inference technique","technique":"diffusion","technique_name":"diffusion","served_params":25823778864,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/vllm-project/vllm/pull/57250","base_model":"google/diffusiongemma-26B-A4B-it","base_url":"https://huggingface.co/google/diffusiongemma-26B-A4B-it","weights_repo":null,"served_checkpoint":"nvidia/diffusiongemma-26B-A4B-it-NVFP4"},"completed":120340,"counts":{"ok":119898},"lifted":false,"calibration":{"acc":0.6205,"conf":0.8525,"ece":0.232,"brier":0.6004,"over95":0.1142,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0026,"conf":0.1666,"acc":0.081},{"lo":0.2,"w":0.0139,"conf":0.2539,"acc":0.1579},{"lo":0.3,"w":0.0206,"conf":0.3586,"acc":0.1896},{"lo":0.4,"w":0.0336,"conf":0.4503,"acc":0.2936},{"lo":0.5,"w":0.0698,"conf":0.5486,"acc":0.3721},{"lo":0.6,"w":0.0679,"conf":0.6504,"acc":0.455},{"lo":0.7,"w":0.0784,"conf":0.7516,"acc":0.4817},{"lo":0.8,"w":0.1138,"conf":0.8561,"acc":0.5411},{"lo":0.9,"w":0.5996,"conf":0.9795,"acc":0.7474}],"areas":{"knowledge":{"ece":0.223,"acc":0.5878,"conf":0.8109},"language":{"ece":0.1665,"acc":0.7462,"conf":0.9127},"retrieval":{"ece":0.4133,"acc":0.4638,"conf":0.8771},"tools":{"ece":0.1201,"acc":0.7218,"conf":0.8355},"arts":{"ece":0.2959,"acc":0.5303,"conf":0.8262}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":37.7,"p95":117.8,"mean":80.7},"results":{"1":{"score":0.9569,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":54.8},"2":{"score":0.4657,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":296.0},"3":{"score":0.5728,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.6903,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":70.4},"5":{"score":0.0201,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":98.3},"6":{"score":0.7988,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":101.8},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.3397,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":62.6},"11":{"score":0.5409,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5693,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":33.7},"20":{"score":0.8906,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":33.3},"21":{"score":0.6024,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":31.1},"22":{"score":0.112,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":127.4},"23":{"score":0.5956,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":53.3},"24":{"score":0.7775,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":32.6},"25":{"score":0.4796,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9806,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":31.7},"27":{"score":0.936,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":32.1},"28":{"score":0.6993,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":30.8},"29":{"score":0.8112,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":36.1},"30":{"score":0.3904,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":35.0},"31":{"score":0.1402,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":60.4},"32":{"score":0.6303,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2994,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":62.4},"36":{"score":0.2319,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5629,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":35.9},"38":{"score":0.1475,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":227.4},"39":{"score":0.8503,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5638,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":30.7,"tracks":[{"track":"A · Arabic","score":0.3514,"headline":false},{"track":"A · English","score":0.5638,"headline":true},{"track":"C · Arabic pairs","score":0.82,"headline":false},{"track":"C · English pairs","score":0.93,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.5923,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":34.0},"42":{"score":0.7406,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":53.7},"43":{"score":0.4719,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.623,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":33.2},"45":{"score":0.1018,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2359,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":55.0},"50":{"score":0.4761,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":57.3},"56":{"score":0.813,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":42.6,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.4176,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":35.7,"edition_added":"v0.2","jev":0.827},"58":{"score":0.6621,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":33.8,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.5181,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":55.2,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.5758,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.7081,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":56.7,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.6837,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":39.4,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":50.9,"balanced_skill":37.25,"breadth_skill":36.47},"frozen_scores":{"balanced_raw":41.51,"balanced_skill":27.8,"breadth_skill":23.93},"categories":[{"id":"knowledge","raw":0.4212,"skill":0.2618,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.6263,"skill":0.4486,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.4834,"skill":0.3702,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.5221,"skill":0.4719,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4603,"skill":0.285,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9569,"skill":0.9418,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.4657,"skill":0.3829,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7989,"skill":0.5257,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7854,"headline":false},{"track":"RouterBench-5shot","score":0.8123,"headline":false}]},"11":{"raw":0.5409,"skill":0.3361,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.8915,"skill":0.783,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6024,"skill":0.2047,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.1073,"skill":0.1003,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5934,"skill":0.1867,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7849,"skill":0.7132,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4796,"skill":0.3061,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.3904,"skill":0.2594,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.4299,"headline":false},{"track":"GSM8K-10choice","score":0.351,"headline":false}]},"31":{"raw":0.1402,"skill":0.0635,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.2319,"skill":0.1311,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.5629,"skill":0.4518,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.5638,"skill":0.4388,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3514,"headline":false},{"track":"A · English","score":0.5638,"headline":true},{"track":"C · Arabic pairs","score":0.82,"headline":false},{"track":"C · English pairs","score":0.93,"headline":false}]},"41":{"raw":0.5923,"skill":0.3885,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4719,"skill":0.1622,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.623,"skill":0.246,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4761,"skill":0.2397,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.813,"skill":0.626,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.4176,"skill":0.345,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6621,"skill":0.5102,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.5181,"skill":0.3575,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.5758,"skill":0.4697,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.7081,"skill":0.3948,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.6837,"skill":0.3674,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.6303,"skill":0.4122,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2994,"skill":0.2901,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1018,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.5693,"skill":0.3549,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.6993,"skill":0.3986,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.8112,"skill":0.7483,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.1475,"skill":0.1202,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8503,"skill":0.7798,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7406,"skill":0.4953,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.6903,"skill":0.6863,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.0201,"skill":0.0142,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.5728,"skill":0.5646,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0564,"skill":0.0564,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"openjev 0.4 API over vLLM 0.29.1rc1.dev551 NVFP4, async scheduling, prefix caching, TRITON_ATTN, one read per question group, one sequential HTTP client"},{"engine":"reflex-27b-v2","name":"reflex 27B","long_name":"reflex Qwen3.8-27B-FP8 (wide choice)","variant":"FP8 · wide choice","meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/kshetrajna12/reflex","base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":"Qwen/Qwen3.8-27B-FP8"},"completed":120340,"counts":{"ok":120340},"lifted":false,"calibration":{"acc":0.7043,"conf":0.7264,"ece":0.0242,"brier":0.3972,"over95":0.0089,"n":35093,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0026,"conf":0.0819,"acc":0.1104},{"lo":0.1,"w":0.0152,"conf":0.1597,"acc":0.2007},{"lo":0.2,"w":0.0259,"conf":0.2518,"acc":0.2239},{"lo":0.3,"w":0.0504,"conf":0.3503,"acc":0.3233},{"lo":0.4,"w":0.0712,"conf":0.4535,"acc":0.3707},{"lo":0.5,"w":0.1404,"conf":0.5491,"acc":0.5409},{"lo":0.6,"w":0.128,"conf":0.6498,"acc":0.6055},{"lo":0.7,"w":0.1149,"conf":0.7502,"acc":0.7531},{"lo":0.8,"w":0.1266,"conf":0.8519,"acc":0.8307},{"lo":0.9,"w":0.3249,"conf":0.9635,"acc":0.9461}],"areas":{"knowledge":{"ece":0.0399,"acc":0.6368,"conf":0.6685},"language":{"ece":0.0096,"acc":0.7813,"conf":0.7778},"retrieval":{"ece":0.099,"acc":0.6871,"conf":0.7856},"tools":{"ece":0.0204,"acc":0.9359,"conf":0.9358},"arts":{"ece":0.0486,"acc":0.589,"conf":0.5883}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":108.3,"p95":1239.7,"mean":256.0},"results":{"1":{"score":0.9711,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":201.8},"2":{"score":0.6333,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":2122.6},"3":{"score":0.8504,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7966,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":206.1},"5":{"score":0.8925,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":329.8},"6":{"score":0.799,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":421.8},"9":{"score":0.5682,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4042,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":255.5},"11":{"score":0.7668,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6446,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":73.6},"20":{"score":0.9584,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":72.7},"21":{"score":0.6328,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":71.7},"22":{"score":0.2845,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1757.9},"23":{"score":0.6518,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":166.8},"24":{"score":0.8231,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":72.6},"25":{"score":0.5051,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9882,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":72.0},"27":{"score":0.9693,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":72.0},"28":{"score":0.7932,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":71.6},"29":{"score":0.9396,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":78.5},"30":{"score":0.6539,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":86.8},"31":{"score":0.1748,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":254.0},"32":{"score":0.6263,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2776,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":298.6},"36":{"score":0.4778,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5244,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":143.1},"38":{"score":0.2003,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1998.2},"39":{"score":0.7825,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5032,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":72.2,"tracks":[{"track":"A · Arabic","score":0.3964,"headline":false},{"track":"A · English","score":0.5032,"headline":true},{"track":"C · Arabic pairs","score":0.77,"headline":false},{"track":"C · English pairs","score":0.965,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.605,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":77.2},"42":{"score":0.8062,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":161.0},"43":{"score":0.6105,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6348,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":72.7},"45":{"score":0.1178,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.1918,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":204.6},"50":{"score":0.4338,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":208.3},"56":{"score":0.7415,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":309.0,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.6189,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":78.5,"edition_added":"v0.2","jev":0.827},"58":{"score":0.7064,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":77.1,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.7656,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":219.9,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.7121,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.7414,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":258.3,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.7073,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":150.6,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":63.84,"balanced_skill":52.16,"breadth_skill":50.69},"frozen_scores":{"balanced_raw":47.19,"balanced_skill":35.36,"breadth_skill":30.0},"categories":[{"id":"knowledge","raw":0.4922,"skill":0.3511,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.6847,"skill":0.5416,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.693,"skill":0.5781,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.7691,"skill":0.7412,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.5481,"skill":0.3965,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9711,"skill":0.961,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6333,"skill":0.5766,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.799,"skill":0.5262,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7854,"headline":false},{"track":"RouterBench-5shot","score":0.8127,"headline":false}]},"11":{"raw":0.7668,"skill":0.6628,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.9604,"skill":0.9207,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6328,"skill":0.2656,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.28,"skill":0.2744,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.6481,"skill":0.2962,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8279,"skill":0.7705,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5051,"skill":0.3401,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.6539,"skill":0.5822,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.7005,"headline":false},{"track":"GSM8K-10choice","score":0.6073,"headline":false}]},"31":{"raw":0.1748,"skill":0.1012,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4778,"skill":0.4092,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.5244,"skill":0.4035,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.5032,"skill":0.3609,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3964,"headline":false},{"track":"A · English","score":0.5032,"headline":true},{"track":"C · Arabic pairs","score":0.77,"headline":false},{"track":"C · English pairs","score":0.965,"headline":false}]},"41":{"raw":0.605,"skill":0.4074,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.6105,"skill":0.3821,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6348,"skill":0.2696,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4338,"skill":0.1782,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.7415,"skill":0.483,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.6189,"skill":0.5714,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.7064,"skill":0.5744,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.7656,"skill":0.6875,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.7121,"skill":0.6401,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.7414,"skill":0.4638,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.7073,"skill":0.4146,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.6263,"skill":0.4059,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2776,"skill":0.268,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1178,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.6446,"skill":0.4676,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.7932,"skill":0.5864,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.9396,"skill":0.9195,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.2003,"skill":0.1747,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.7825,"skill":0.6801,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.8062,"skill":0.623,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.7966,"skill":0.794,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.8925,"skill":0.8919,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.8504,"skill":0.8475,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.5682,"skill":0.5682,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.2328,"skill":0.2328,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author reflex frontend (serving-source archive 1dfaa87f: d08a8c6 + admission/readout overlays, wide_choice.py) over SGLang 0.5.20 from the pinned image b27fce60, --permutations 2, pressure admission 6, published SGLang flags with the triton attention backend (fa3 is Hopper-only), one sequential client, one RTX PRO 6000 HF Job","new_in_edition":true},{"engine":"rune-26b-a4b-v3","name":"Surogate Rune 26B-A4B v3","long_name":"Surogate Rune 26B-A4B v3","variant":"bf16","meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":25805936206,"params_estimated":false,"params_from_name":false,"trained_bytes":51612008580,"model_url":"https://huggingface.co/surogate/rune-26b-a4b-GGUF","code_url":"https://github.com/invergent-ai/surogate","base_model":"google/gemma-4-26B-A4B-it","base_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","weights_repo":"surogate/rune-26b-a4b-GGUF","served_checkpoint":"bd4a7cbbed66af1a95dfbac12cabf52e6e677411"},"completed":120340,"counts":{"ok":119897,"error":1},"lifted":false,"calibration":{"acc":0.7476,"conf":0.8672,"ece":0.1196,"brier":0.3704,"over95":0.0482,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0001,"conf":0.0918,"acc":0.0},{"lo":0.1,"w":0.0063,"conf":0.1638,"acc":0.0665},{"lo":0.2,"w":0.0109,"conf":0.2533,"acc":0.1317},{"lo":0.3,"w":0.0224,"conf":0.3525,"acc":0.2547},{"lo":0.4,"w":0.0307,"conf":0.454,"acc":0.3172},{"lo":0.5,"w":0.059,"conf":0.5486,"acc":0.422},{"lo":0.6,"w":0.055,"conf":0.6485,"acc":0.4686},{"lo":0.7,"w":0.0684,"conf":0.7551,"acc":0.5358},{"lo":0.8,"w":0.0918,"conf":0.8543,"acc":0.6461},{"lo":0.9,"w":0.6553,"conf":0.9819,"acc":0.8906}],"areas":{"knowledge":{"ece":0.1302,"acc":0.673,"conf":0.8032},"language":{"ece":0.0787,"acc":0.8334,"conf":0.9121},"retrieval":{"ece":0.1446,"acc":0.7601,"conf":0.9018},"tools":{"ece":0.0574,"acc":0.915,"conf":0.9603},"arts":{"ece":0.1981,"acc":0.6464,"conf":0.8434}}},"gaps":{"share":0.0,"reasons":[{"reason":"runtime error","share":0.0}]},"latency":{"median":120.5,"p95":682.8,"mean":267.2},"results":{"1":{"score":0.9481,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":250.8},"2":{"score":0.6439,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1890.7},"3":{"score":0.8327,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8375,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":289.4},"5":{"score":0.874,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":384.0},"6":{"score":0.7987,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":572.4},"9":{"score":0.4659,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.664,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":366.1},"11":{"score":0.7697,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7352,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":83.9},"20":{"score":0.8956,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":84.2},"21":{"score":0.6221,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":56.9},"22":{"score":0.668,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1981.2},"23":{"score":0.6314,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":172.7},"24":{"score":0.8391,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":85.3},"25":{"score":0.4694,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9903,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":85.1},"27":{"score":0.9684,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":85.0},"28":{"score":0.8595,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":53.3},"29":{"score":0.924,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":115.3},"30":{"score":0.7972,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":85.5},"31":{"score":0.1874,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":243.8},"32":{"score":0.6489,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3485,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":315.5},"36":{"score":0.4632,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5529,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":118.4},"38":{"score":0.2676,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1802.5},"39":{"score":0.8843,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.6038,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":84.6,"tracks":[{"track":"A · Arabic","score":0.5258,"headline":false},{"track":"A · English","score":0.6038,"headline":true},{"track":"C · Arabic pairs","score":0.88,"headline":false},{"track":"C · English pairs","score":0.97,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.7662,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":85.8},"42":{"score":0.8044,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":177.3},"43":{"score":0.7439,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.7274,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":84.0},"45":{"score":0.106,"jev":0.204,"metric":"accuracy","requests":501,"answered":500,"unsupported":0,"errors":1,"median_ms":null},"48":{"score":0.2028,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":240.7},"50":{"score":0.4224,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":273.1},"56":{"score":0.809,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":314.3,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.672,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":86.7,"edition_added":"v0.2","jev":0.827},"58":{"score":0.8115,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":90.5,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.7601,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":229.4,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.7405,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.768,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":241.8,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.8067,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":148.8,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":67.3,"balanced_skill":57.44,"breadth_skill":56.47},"frozen_scores":{"balanced_raw":50.75,"balanced_skill":39.83,"breadth_skill":33.25},"categories":[{"id":"knowledge","raw":0.5485,"skill":0.4336,"coverage":0.9998,"pending":0,"n":10},{"id":"language","raw":0.744,"skill":0.6306,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.724,"skill":0.6352,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.742,"skill":0.7122,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.583,"skill":0.419,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9481,"skill":0.9299,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6439,"skill":0.5887,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7987,"skill":0.5251,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7864,"headline":false},{"track":"RouterBench-5shot","score":0.8111,"headline":false}]},"11":{"raw":0.7697,"skill":0.667,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.8996,"skill":0.7991,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6221,"skill":0.2443,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.6607,"skill":0.6581,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.6256,"skill":0.2513,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8447,"skill":0.793,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4694,"skill":0.2925,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.7972,"skill":0.7571,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.8423,"headline":false},{"track":"GSM8K-10choice","score":0.7521,"headline":false}]},"31":{"raw":0.1874,"skill":0.115,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4632,"skill":0.3928,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.5529,"skill":0.4393,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.6038,"skill":0.4903,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5258,"headline":false},{"track":"A · English","score":0.6038,"headline":true},{"track":"C · Arabic pairs","score":0.88,"headline":false},{"track":"C · English pairs","score":0.97,"headline":false}]},"41":{"raw":0.7662,"skill":0.6493,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.7439,"skill":0.5936,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.7274,"skill":0.4548,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4224,"skill":0.1618,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.809,"skill":0.618,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.672,"skill":0.6311,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.8115,"skill":0.7268,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.7601,"skill":0.6801,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.7405,"skill":0.6756,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.768,"skill":0.519,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.8067,"skill":0.6134,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.6489,"skill":0.4418,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.3485,"skill":0.3399,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1058,"skill":0.0,"coverage":0.998,"random":0.1641,"tracks":[]},"12":{"raw":0.7352,"skill":0.6034,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.8595,"skill":0.719,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.924,"skill":0.8987,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.2676,"skill":0.2442,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8843,"skill":0.8298,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.8044,"skill":0.6195,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.8375,"skill":0.8354,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.874,"skill":0.8732,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.8327,"skill":0.8295,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.4659,"skill":0.4659,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.1888,"skill":0.1888,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"surogate v1.5.3 release wheel (pinned rune-v3-audit setup: uv py3.12.14, torch 2.11.0+cu130, noble glibc shim) serving the rune-v3 bf16 safetensors, surogate serve --max-model-len 262144 --kv-capacity 262144 --max-num-seqs 8, POST /v1/decisions with the whole row in one request (shared state prefill), one sequential client via run.py (latency_run_extra.py wrapper, engine latency_rune.RuneV3Latency)","new_in_edition":true},{"engine":"semif","name":"SemIf","long_name":"SemIf Qwen3.5-4B Base","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/TheoLeeCJ/openjev","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":null,"served_checkpoint":null},"completed":120340,"counts":{"ok":105023,"unsupported":15317},"lifted":true,"calibration":{"acc":0.6404,"conf":0.7393,"ece":0.0989,"brier":0.4825,"over95":0.0256,"n":70067,"benchmarks":28,"rel":[{"lo":0.0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0004,"conf":0.1774,"acc":0.0},{"lo":0.2,"w":0.0171,"conf":0.263,"acc":0.1845},{"lo":0.3,"w":0.057,"conf":0.3561,"acc":0.2879},{"lo":0.4,"w":0.0706,"conf":0.4498,"acc":0.324},{"lo":0.5,"w":0.1669,"conf":0.5495,"acc":0.5089},{"lo":0.6,"w":0.1125,"conf":0.6493,"acc":0.5585},{"lo":0.7,"w":0.1206,"conf":0.751,"acc":0.616},{"lo":0.8,"w":0.1155,"conf":0.8532,"acc":0.6777},{"lo":0.9,"w":0.3393,"conf":0.9689,"acc":0.877}],"areas":{"knowledge":{"ece":0.1033,"acc":0.5598,"conf":0.6631},"language":{"ece":0.1291,"acc":0.6952,"conf":0.8215},"retrieval":{"ece":0.1814,"acc":0.6173,"conf":0.7888},"tools":{"ece":0.0383,"acc":0.919,"conf":0.8897},"arts":{"ece":0.0532,"acc":0.611,"conf":0.6642}}},"gaps":{"share":0.1273,"reasons":[{"reason":"supports only 2–16 answer options per question","share":0.1273}]},"latency":{"median":113.1,"p95":214.6,"mean":220.5},"results":{"1":{"score":0.9327,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":119.4},"2":{"score":0.5961,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":9594.1},"3":{"score":null,"jev":null,"metric":null,"requests":508,"answered":0,"unsupported":508,"errors":0,"median_ms":null},"4":{"score":null,"jev":null,"metric":null,"requests":3080,"answered":0,"unsupported":3080,"errors":0,"median_ms":null},"5":{"score":null,"jev":null,"metric":null,"requests":5500,"answered":0,"unsupported":5500,"errors":0,"median_ms":null},"6":{"score":0.7954,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":202.7},"9":{"score":0.0909,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.568,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":119.0},"11":{"score":0.5951,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5491,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":111.5},"20":{"score":0.8522,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":110.9},"21":{"score":0.5993,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":110.7},"22":{"score":null,"jev":null,"metric":null,"requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"23":{"score":0.566,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":113.0},"24":{"score":0.7144,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":111.4},"25":{"score":0.3673,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9676,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":111.1},"27":{"score":0.9181,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":111.1},"28":{"score":0.6598,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":110.6},"29":{"score":0.8272,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":112.4},"30":{"score":0.3131,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":112.9},"31":{"score":0.3316,"jev":0.3914,"metric":"accuracy","requests":5000,"answered":787,"unsupported":4213,"errors":0,"median_ms":null},"32":{"score":0.5465,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":140.7},"36":{"score":0.4143,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3616,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":114.1},"38":{"score":0.135,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":467.1},"39":{"score":0.6596,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.39,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":110.5,"tracks":[{"track":"A · Arabic","score":0.281,"headline":false},{"track":"A · English","score":0.39,"headline":true},{"track":"C · Arabic pairs","score":0.645,"headline":false},{"track":"C · English pairs","score":0.84,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4396,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":111.6},"42":{"score":0.7196,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":113.3},"43":{"score":0.3895,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5596,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":110.7},"45":{"score":0.1107,"jev":0.2049,"metric":"accuracy","requests":501,"answered":488,"unsupported":13,"errors":0,"median_ms":null},"48":{"score":0.2252,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":115.9},"50":{"score":0.4433,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":72.6},"56":{"score":0.6005,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":128.2,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.4368,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":115.1,"edition_added":"v0.2","jev":0.827},"58":{"score":0.5756,"metric":"accuracy","requests":5507,"answered":5332,"unsupported":175,"errors":0,"median_ms":112.3,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.5468,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":118.2,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.5777,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.5344,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":115.8,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.6162,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":57.5,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":42.21,"balanced_skill":25.94,"breadth_skill":24.75},"frozen_scores":{"balanced_raw":38.13,"balanced_skill":22.81,"breadth_skill":19.3},"categories":[{"id":"knowledge","raw":0.3541,"skill":0.171,"coverage":0.9156,"pending":0,"n":10},{"id":"language","raw":0.5582,"skill":0.3379,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.3145,"skill":0.1572,"coverage":0.6364,"pending":0,"n":6},{"id":"tools","raw":0.4357,"skill":0.391,"coverage":0.7778,"pending":0,"n":5},{"id":"arts","raw":0.4369,"skill":0.2492,"coverage":0.8611,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9327,"skill":0.9092,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.5961,"skill":0.5335,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7954,"skill":0.5172,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7828,"headline":false},{"track":"RouterBench-5shot","score":0.8079,"headline":false}]},"11":{"raw":0.5951,"skill":0.4144,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.8517,"skill":0.7034,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5993,"skill":0.1986,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0,"skill":0.0,"coverage":0,"random":0.0078,"tracks":[]},"23":{"raw":0.5545,"skill":0.109,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7275,"skill":0.6367,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3673,"skill":0.1565,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.3131,"skill":0.1702,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.4003,"headline":false},{"track":"GSM8K-10choice","score":0.2259,"headline":false}]},"31":{"raw":0.0522,"skill":0.0,"coverage":0.1574,"random":0.0819,"tracks":[]},"36":{"raw":0.4143,"skill":0.3374,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.3616,"skill":0.1992,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.39,"skill":0.2152,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.281,"headline":false},{"track":"A · English","score":0.39,"headline":true},{"track":"C · Arabic pairs","score":0.645,"headline":false},{"track":"C · English pairs","score":0.84,"headline":false}]},"41":{"raw":0.4396,"skill":0.1595,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3895,"skill":0.0313,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5596,"skill":0.1192,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4433,"skill":0.1921,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.6005,"skill":0.201,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.4368,"skill":0.3666,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.5573,"skill":0.3583,"coverage":0.9682,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.5468,"skill":0.3957,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.5777,"skill":0.4721,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.5344,"skill":0.0346,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.6162,"skill":0.2324,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5465,"skill":0.279,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2,"skill":0.1894,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1078,"skill":0.0,"coverage":0.9741,"random":0.1641,"tracks":[]},"12":{"raw":0.5491,"skill":0.3246,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.6598,"skill":0.3196,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.8272,"skill":0.7696,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.135,"skill":0.1073,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.6596,"skill":0.4993,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7196,"skill":0.4545,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0127,"tracks":[]},"5":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.006,"tracks":[]},"3":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0909,"skill":0.0909,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0992,"skill":0.0992,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.6863,"pending":0.24,"latency_rows":673,"latency_path":"author score_shared with shared prefix, bf16, causal-conv1d 1.7.0 CUDA kernel (JEV_SEMIF_REQUIRE_CUDA_CONV=1), board lifts; fleet venv torch 2.14.0+cu130"},{"engine":"solomon-v11-bf16-encoding","name":"Solomon v1.1","long_name":"Solomon v1.1","variant":null,"meta":{"kind":"head / adapter","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":903468342,"model_url":"https://huggingface.co/DoccyHealth/Solomon","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":"DoccyHealth/Solomon","served_checkpoint":null},"completed":120340,"counts":{"ok":92830,"unsupported":27498},"lifted":false,"calibration":{"acc":0.6907,"conf":0.7415,"ece":0.0807,"brier":0.4423,"over95":0.0302,"n":69332,"benchmarks":27,"rel":[{"lo":0.0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0046,"conf":0.1867,"acc":0.3613},{"lo":0.2,"w":0.0128,"conf":0.2605,"acc":0.2044},{"lo":0.3,"w":0.0956,"conf":0.3491,"acc":0.4359},{"lo":0.4,"w":0.0721,"conf":0.4484,"acc":0.5093},{"lo":0.5,"w":0.1317,"conf":0.5513,"acc":0.5625},{"lo":0.6,"w":0.1045,"conf":0.6462,"acc":0.621},{"lo":0.7,"w":0.0818,"conf":0.751,"acc":0.653},{"lo":0.8,"w":0.1018,"conf":0.8521,"acc":0.7032},{"lo":0.9,"w":0.3952,"conf":0.97,"acc":0.8706}],"areas":{"knowledge":{"ece":0.0999,"acc":0.6208,"conf":0.6241},"language":{"ece":0.0728,"acc":0.8105,"conf":0.8641},"retrieval":{"ece":0.2617,"acc":0.5834,"conf":0.8451},"tools":{"ece":0.0072,"acc":0.991,"conf":0.9929},"arts":{"ece":0.0823,"acc":0.5919,"conf":0.6736}}},"gaps":{"share":0.2285,"reasons":[{"reason":"supports only 2–8 answer options per question","share":0.2251},{"reason":"request shape outside the model's schema","share":0.0034},{"reason":"request larger than its native request budget","share":0.0}]},"latency":{"median":223.9,"p95":1176.8,"mean":503.2},"results":{"1":{"score":0.9799,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":434.6},"2":{"score":0.6558,"jev":0.6532,"metric":"nDCG@10","requests":685,"answered":683,"unsupported":2,"errors":0,"median_ms":4192.5},"3":{"score":null,"jev":null,"metric":null,"requests":508,"answered":0,"unsupported":508,"errors":0,"median_ms":null},"4":{"score":null,"jev":null,"metric":null,"requests":3080,"answered":0,"unsupported":3080,"errors":0,"median_ms":null},"5":{"score":null,"jev":null,"metric":null,"requests":5500,"answered":0,"unsupported":5500,"errors":0,"median_ms":null},"6":{"score":null,"jev":null,"metric":null,"requests":10000,"answered":0,"unsupported":10000,"errors":0,"median_ms":null},"9":{"score":null,"jev":null,"metric":null,"requests":88,"answered":0,"unsupported":88,"errors":0,"median_ms":null},"10":{"score":0.592,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":478.6},"11":{"score":0.7153,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6708,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":220.6},"20":{"score":0.9414,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":219.4},"21":{"score":0.5422,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":217.3},"22":{"score":null,"jev":null,"metric":null,"requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"23":{"score":0.502,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":258.4},"24":{"score":0.6995,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":218.6},"25":{"score":0.4286,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.8657,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":219.1},"27":{"score":0.8456,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":218.5},"28":{"score":0.8106,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":215.9},"29":{"score":0.9099,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":221.6},"30":{"score":null,"jev":null,"metric":null,"requests":2638,"answered":1319,"unsupported":1319,"errors":0,"median_ms":220.4},"31":{"score":0.4019,"jev":0.4533,"metric":"accuracy","requests":5000,"answered":525,"unsupported":4475,"errors":0,"median_ms":null},"32":{"score":0.6064,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2055,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":1075.6},"36":{"score":0.4688,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5618,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":224.5},"38":{"score":0.2068,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":7732.0},"39":{"score":0.9186,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4822,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":221.7,"tracks":[{"track":"A · Arabic","score":0.4583,"headline":false},{"track":"A · English","score":0.4822,"headline":true},{"track":"C · Arabic pairs","score":0.715,"headline":false},{"track":"C · English pairs","score":0.9,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.694,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":219.5},"42":{"score":0.8176,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":257.8},"43":{"score":0.6933,"jev":0.734,"metric":"accuracy","requests":570,"answered":564,"unsupported":6,"errors":0,"median_ms":null},"44":{"score":0.7044,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":219.9},"45":{"score":0.1474,"jev":0.2138,"metric":"accuracy","requests":501,"answered":407,"unsupported":94,"errors":0,"median_ms":null},"48":{"score":0.2385,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":282.7},"50":{"score":0.3729,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":268.7},"56":{"score":0.6815,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1181.1,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.7072,"metric":"accuracy","requests":12032,"answered":1250,"unsupported":10782,"errors":0,"median_ms":219.0,"edition_added":"v0.2","jev":0.827},"58":{"score":0.7254,"metric":"accuracy","requests":5507,"answered":5081,"unsupported":426,"errors":0,"median_ms":220.1,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.8264,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":295.8,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.517,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.7552,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":286.7,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.711,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":234.5,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":47.98,"balanced_skill":36.43,"breadth_skill":34.09},"frozen_scores":{"balanced_raw":38.53,"balanced_skill":27.33,"breadth_skill":22.43},"categories":[{"id":"knowledge","raw":0.3513,"skill":0.2213,"coverage":0.6946,"pending":0,"n":10},{"id":"language","raw":0.7037,"skill":0.5728,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.3813,"skill":0.2597,"coverage":0.6364,"pending":0,"n":6},{"id":"tools","raw":0.4919,"skill":0.4697,"coverage":0.5921,"pending":0,"n":5},{"id":"arts","raw":0.4078,"skill":0.2116,"coverage":0.8611,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9799,"skill":0.9729,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6539,"skill":0.6003,"coverage":0.9971,"random":0.1341,"tracks":[]},"6":{"raw":0.0,"skill":0.0,"coverage":0,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.0,"headline":false},{"track":"RouterBench-5shot","score":0.0,"headline":false}]},"11":{"raw":0.7153,"skill":0.5882,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.9437,"skill":0.8874,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5422,"skill":0.0845,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0,"skill":0.0,"coverage":0,"random":0.0078,"tracks":[]},"23":{"raw":0.505,"skill":0.01,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6953,"skill":0.5938,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4286,"skill":0.2381,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.0,"skill":0.0,"coverage":0,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.0,"headline":false},{"track":"GSM8K-10choice","score":0.0,"headline":false}]},"31":{"raw":0.0422,"skill":0.0,"coverage":0.105,"random":0.0819,"tracks":[]},"36":{"raw":0.4688,"skill":0.399,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.5618,"skill":0.4504,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.4822,"skill":0.3338,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4583,"headline":false},{"track":"A · English","score":0.4822,"headline":true},{"track":"C · Arabic pairs","score":0.715,"headline":false},{"track":"C · English pairs","score":0.9,"headline":false}]},"41":{"raw":0.694,"skill":0.541,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.686,"skill":0.5018,"coverage":0.9895,"random":0.3697,"tracks":[]},"44":{"raw":0.7044,"skill":0.4088,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3729,"skill":0.0899,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.6815,"skill":0.363,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.0735,"skill":0.0,"coverage":0.1039,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6693,"skill":0.5207,"coverage":0.9226,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.8264,"skill":0.7685,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.517,"skill":0.3962,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.7552,"skill":0.4924,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.711,"skill":0.422,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.6064,"skill":0.3742,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2055,"skill":0.195,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1197,"skill":0.0,"coverage":0.8124,"random":0.1641,"tracks":[]},"12":{"raw":0.6708,"skill":0.5069,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.8106,"skill":0.6212,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.9099,"skill":0.8799,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.2068,"skill":0.1814,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.9186,"skill":0.8803,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.8176,"skill":0.6451,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0127,"tracks":[]},"5":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.006,"tracks":[]},"3":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0,"tracks":[]},"48":{"raw":0.046,"skill":0.046,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.6037,"pending":0.24,"latency_rows":562,"latency_path":"Solomon v1.1 bf16 serving binding with the encoding-fix path (solomon-v11-bf16-empty-context engine), sdpa (flash-attn optional, absent), venv-solomon torch 2.13.0+cu130 / transformers 5.17.0 / fla 0.5.2 as the binding requires"},{"engine":"tev1-0.8b","name":"Tev1-0.8B-experimental","long_name":"Together Tev1 Qwen3.5-0.8B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":873438784,"params_estimated":false,"params_from_name":false,"trained_bytes":1746942600,"model_url":"https://huggingface.co/togethercomputer/Tev1-0.8B-experimental","code_url":"https://github.com/togethercomputer/tev1","base_model":"Qwen/Qwen3.5-0.8B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B-Base","weights_repo":"togethercomputer/Tev1-0.8B-experimental","served_checkpoint":null},"completed":120340,"counts":{"ok":105156,"unsupported":14732},"lifted":false,"calibration":{"acc":0.5022,"conf":0.6279,"ece":0.1257,"brier":0.6067,"over95":0.012,"n":32652,"benchmarks":28,"rel":[{"lo":0.0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0025,"conf":0.1873,"acc":0.0958},{"lo":0.2,"w":0.0409,"conf":0.2605,"acc":0.1139},{"lo":0.3,"w":0.0888,"conf":0.3533,"acc":0.269},{"lo":0.4,"w":0.1141,"conf":0.4501,"acc":0.354},{"lo":0.5,"w":0.2462,"conf":0.5522,"acc":0.463},{"lo":0.6,"w":0.1575,"conf":0.6467,"acc":0.525},{"lo":0.7,"w":0.1341,"conf":0.7483,"acc":0.6048},{"lo":0.8,"w":0.0969,"conf":0.8523,"acc":0.6472},{"lo":0.9,"w":0.1189,"conf":0.9528,"acc":0.7778}],"areas":{"knowledge":{"ece":0.1353,"acc":0.4149,"conf":0.5502},"language":{"ece":0.1351,"acc":0.5196,"conf":0.6547},"retrieval":{"ece":0.1907,"acc":0.6214,"conf":0.8078},"tools":{"ece":0.0341,"acc":0.7799,"conf":0.7678},"arts":{"ece":0.1187,"acc":0.5044,"conf":0.6213}}},"gaps":{"share":0.1229,"reasons":[{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'q1' has 151","share":0.0459},{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'q1' has 77","share":0.0257},{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'chord' has 129","share":0.0167}]},"latency":{"median":28.5,"p95":133.9,"mean":50.1},"results":{"1":{"score":0.6612,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":63.9},"2":{"score":0.5723,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":195.4},"3":{"score":null,"jev":null,"metric":null,"requests":508,"answered":0,"unsupported":508,"errors":0,"median_ms":null},"4":{"score":null,"jev":null,"metric":null,"requests":3080,"answered":0,"unsupported":3080,"errors":0,"median_ms":null},"5":{"score":null,"jev":null,"metric":null,"requests":5500,"answered":0,"unsupported":5500,"errors":0,"median_ms":null},"6":{"score":0.5868,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":131.6},"9":{"score":0.0114,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5968,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":61.5},"11":{"score":0.4607,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.3761,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":27.6},"20":{"score":0.5072,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":27.4},"21":{"score":0.5126,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":27.3},"22":{"score":null,"jev":null,"metric":null,"requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"23":{"score":0.537,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":59.3},"24":{"score":0.4591,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":27.6},"25":{"score":0.2755,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.8199,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":27.4},"27":{"score":0.6195,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":27.5},"28":{"score":0.5225,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":27.4},"29":{"score":0.4481,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":28.0},"30":{"score":0.2214,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":27.8},"31":{"score":0.195,"jev":0.2929,"metric":"accuracy","requests":5000,"answered":1359,"unsupported":3641,"errors":0,"median_ms":59.2},"32":{"score":0.5199,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0582,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":78.9},"36":{"score":0.2936,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.2181,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":28.9},"38":{"score":0.0538,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":171.0},"39":{"score":0.5778,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2126,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":27.7,"tracks":[{"track":"A · Arabic","score":0.2317,"headline":false},{"track":"A · English","score":0.2126,"headline":true},{"track":"C · Arabic pairs","score":0.505,"headline":false},{"track":"C · English pairs","score":0.465,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4464,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":27.8},"42":{"score":0.613,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":59.9},"43":{"score":0.3807,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5052,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":27.5},"45":{"score":0.1486,"jev":0.2028,"metric":"accuracy","requests":501,"answered":498,"unsupported":3,"errors":0,"median_ms":null},"48":{"score":0.2466,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":60.4},"50":{"score":0.3998,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":61.2},"56":{"score":0.5,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":70.5,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.2288,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":28.0,"edition_added":"v0.2","jev":0.827},"58":{"score":0.4362,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":27.9,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.6681,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":60.5,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.3068,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.1825,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":60.9,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5308,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":31.9,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":32.37,"balanced_skill":12.85,"breadth_skill":11.27},"frozen_scores":{"balanced_raw":29.8,"balanced_skill":9.37,"breadth_skill":8.06},"categories":[{"id":"knowledge","raw":0.2819,"skill":0.0729,"coverage":0.9319,"pending":0,"n":10},{"id":"language","raw":0.3927,"skill":0.1405,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.2426,"skill":0.0488,"coverage":0.6364,"pending":0,"n":6},{"id":"tools","raw":0.3787,"skill":0.3197,"coverage":0.7778,"pending":0,"n":5},{"id":"arts","raw":0.315,"skill":0.0512,"coverage":0.8611,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.6612,"skill":0.5426,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.5723,"skill":0.5061,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.5869,"skill":0.0362,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.5589,"headline":false},{"track":"RouterBench-5shot","score":0.6148,"headline":false}]},"11":{"raw":0.4607,"skill":0.2201,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.5108,"skill":0.0215,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5126,"skill":0.0251,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0,"skill":0.0,"coverage":0,"random":0.0078,"tracks":[]},"23":{"raw":0.5216,"skill":0.0431,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4774,"skill":0.3032,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2755,"skill":0.034,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.2214,"skill":0.0582,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.3101,"headline":false},{"track":"GSM8K-10choice","score":0.1327,"headline":false}]},"31":{"raw":0.053,"skill":0.0,"coverage":0.2718,"random":0.0819,"tracks":[]},"36":{"raw":0.2936,"skill":0.2009,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.2181,"skill":0.0192,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.2126,"skill":0.0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2317,"headline":false},{"track":"A · English","score":0.2126,"headline":true},{"track":"C · Arabic pairs","score":0.505,"headline":false},{"track":"C · English pairs","score":0.465,"headline":false}]},"41":{"raw":0.4464,"skill":0.1696,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3807,"skill":0.0174,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5052,"skill":0.0104,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3998,"skill":0.1289,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.2288,"skill":0.1326,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.4362,"skill":0.1828,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.6681,"skill":0.5575,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.3068,"skill":0.1335,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.1825,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5308,"skill":0.0616,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5199,"skill":0.2367,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0582,"skill":0.0457,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1477,"skill":0.0,"coverage":0.994,"random":0.1641,"tracks":[]},"12":{"raw":0.3761,"skill":0.0655,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5225,"skill":0.045,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.4481,"skill":0.2641,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0538,"skill":0.0235,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.5778,"skill":0.379,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.613,"skill":0.2471,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0127,"tracks":[]},"5":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.006,"tracks":[]},"3":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0114,"skill":0.0114,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0136,"skill":0.0136,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.6909,"pending":0.24,"latency_rows":677,"latency_path":"in-process vLLM 0.29.0 LLM.generate with all questions of the row in one call, CUDA graphs, gpu util 0.9 (single process)","new_in_edition":true},{"engine":"tev1-4b","name":"Tev1-4B-experimental","long_name":"Together Tev1 Qwen3.5-4B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":9319828096,"model_url":"https://huggingface.co/togethercomputer/Tev1-4B-experimental","code_url":"https://github.com/togethercomputer/tev1","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"togethercomputer/Tev1-4B-experimental","served_checkpoint":null},"completed":120340,"counts":{"ok":105166,"unsupported":14732},"lifted":false,"calibration":{"acc":0.6345,"conf":0.7386,"ece":0.104,"brier":0.4763,"over95":0.0174,"n":32652,"benchmarks":28,"rel":[{"lo":0.0,"w":0.0002,"conf":0.0919,"acc":0.0},{"lo":0.1,"w":0.0124,"conf":0.1624,"acc":0.11},{"lo":0.2,"w":0.0224,"conf":0.2506,"acc":0.1198},{"lo":0.3,"w":0.0507,"conf":0.352,"acc":0.1994},{"lo":0.4,"w":0.0557,"conf":0.4561,"acc":0.3715},{"lo":0.5,"w":0.1476,"conf":0.5511,"acc":0.4759},{"lo":0.6,"w":0.1197,"conf":0.6492,"acc":0.5373},{"lo":0.7,"w":0.1334,"conf":0.7497,"acc":0.606},{"lo":0.8,"w":0.1248,"conf":0.8522,"acc":0.7026},{"lo":0.9,"w":0.3331,"conf":0.9674,"acc":0.8904}],"areas":{"knowledge":{"ece":0.1215,"acc":0.5575,"conf":0.6791},"language":{"ece":0.0705,"acc":0.7148,"conf":0.785},"retrieval":{"ece":0.2745,"acc":0.4965,"conf":0.7693},"tools":{"ece":0.0258,"acc":0.9313,"conf":0.9228},"arts":{"ece":0.1059,"acc":0.5864,"conf":0.6903}}},"gaps":{"share":0.1229,"reasons":[{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'q1' has 151","share":0.0459},{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'q1' has 77","share":0.0257},{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'chord' has 129","share":0.0167}]},"latency":{"median":35.8,"p95":166.0,"mean":67.6},"results":{"1":{"score":0.902,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":83.8},"2":{"score":0.6501,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":452.0},"3":{"score":null,"jev":null,"metric":null,"requests":508,"answered":0,"unsupported":508,"errors":0,"median_ms":null},"4":{"score":null,"jev":null,"metric":null,"requests":3080,"answered":0,"unsupported":3080,"errors":0,"median_ms":null},"5":{"score":null,"jev":null,"metric":null,"requests":5500,"answered":0,"unsupported":5500,"errors":0,"median_ms":null},"6":{"score":0.6781,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":163.8},"9":{"score":0.1477,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.407,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":77.0},"11":{"score":0.6772,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5639,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":34.4},"20":{"score":0.7662,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":34.3},"21":{"score":0.583,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":33.6},"22":{"score":null,"jev":null,"metric":null,"requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"23":{"score":0.5844,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":74.6},"24":{"score":0.7227,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":34.3},"25":{"score":0.4235,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.976,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":33.8},"27":{"score":0.9215,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":33.8},"28":{"score":0.6961,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":34.1},"29":{"score":0.8193,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":35.0},"30":{"score":0.3666,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":35.0},"31":{"score":0.22,"jev":0.2929,"metric":"accuracy","requests":5000,"answered":1359,"unsupported":3641,"errors":0,"median_ms":73.7},"32":{"score":0.5798,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.203,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":120.4},"36":{"score":0.4302,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4421,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":36.0},"38":{"score":0.1032,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":433.6},"39":{"score":0.8631,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.3594,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":34.4,"tracks":[{"track":"A · Arabic","score":0.3776,"headline":false},{"track":"A · English","score":0.3594,"headline":true},{"track":"C · Arabic pairs","score":0.57,"headline":false},{"track":"C · English pairs","score":0.89,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4766,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":34.7},"42":{"score":0.7734,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":74.7},"43":{"score":0.5018,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.631,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":34.2},"45":{"score":0.1165,"jev":0.2028,"metric":"accuracy","requests":501,"answered":498,"unsupported":3,"errors":0,"median_ms":null},"48":{"score":0.1997,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":75.5},"50":{"score":0.4159,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":76.4},"56":{"score":0.5065,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":107.0,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.4537,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":35.2,"edition_added":"v0.2","jev":0.827},"58":{"score":0.6348,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":35.0,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.6054,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":76.0,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.5549,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.5033,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":75.9,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.641,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":40.1,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":44.66,"balanced_skill":29.24,"breadth_skill":27.9},"frozen_scores":{"balanced_raw":39.39,"balanced_skill":23.94,"breadth_skill":20.16},"categories":[{"id":"knowledge","raw":0.3977,"skill":0.2349,"coverage":0.9319,"pending":0,"n":10},{"id":"language","raw":0.5898,"skill":0.3919,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.3191,"skill":0.1548,"coverage":0.6364,"pending":0,"n":6},{"id":"tools","raw":0.4603,"skill":0.4183,"coverage":0.7778,"pending":0,"n":5},{"id":"arts","raw":0.4331,"skill":0.2292,"coverage":0.8611,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.902,"skill":0.8677,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6501,"skill":0.5959,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.6781,"skill":0.2394,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.6627,"headline":false},{"track":"RouterBench-5shot","score":0.6935,"headline":false}]},"11":{"raw":0.6772,"skill":0.5332,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.7699,"skill":0.5399,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.583,"skill":0.1659,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0,"skill":0.0,"coverage":0,"random":0.0078,"tracks":[]},"23":{"raw":0.5535,"skill":0.107,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7375,"skill":0.65,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4235,"skill":0.2313,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.3666,"skill":0.2365,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.4632,"headline":false},{"track":"GSM8K-10choice","score":0.2699,"headline":false}]},"31":{"raw":0.0598,"skill":0.0,"coverage":0.2718,"random":0.0819,"tracks":[]},"36":{"raw":0.4302,"skill":0.3554,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.4421,"skill":0.3002,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.3594,"skill":0.1759,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3776,"headline":false},{"track":"A · English","score":0.3594,"headline":true},{"track":"C · Arabic pairs","score":0.57,"headline":false},{"track":"C · English pairs","score":0.89,"headline":false}]},"41":{"raw":0.4766,"skill":0.2149,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.5018,"skill":0.2095,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.631,"skill":0.262,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4159,"skill":0.1522,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5065,"skill":0.013,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.4537,"skill":0.3856,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6348,"skill":0.4706,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.6054,"skill":0.4739,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.5549,"skill":0.4436,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.5033,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.641,"skill":0.282,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5798,"skill":0.332,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.203,"skill":0.1924,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1158,"skill":0.0,"coverage":0.994,"random":0.1641,"tracks":[]},"12":{"raw":0.5639,"skill":0.3468,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.6961,"skill":0.3922,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.8193,"skill":0.7591,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.1032,"skill":0.0745,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8631,"skill":0.7986,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7734,"skill":0.5591,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0127,"tracks":[]},"5":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.006,"tracks":[]},"3":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0189,"tracks":[]},"9":{"raw":0.1477,"skill":0.1477,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.2012,"skill":0.2012,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.6909,"pending":0.24,"latency_rows":677,"latency_path":"in-process vLLM 0.29.0 LLM.generate with all questions of the row in one call, CUDA graphs, gpu util 0.9 (single process)","new_in_edition":true},{"engine":"this-that-1.2","name":"this-that 1.2","long_name":"FLock this-that 1.2","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":1881825088,"params_estimated":false,"params_from_name":false,"trained_bytes":3763692048,"model_url":"https://huggingface.co/flock-io/this-that-model-1.2","code_url":"https://github.com/FLock-io/this-that-model","base_model":"Mapika/decider-2b","base_url":"https://huggingface.co/Mapika/decider-2b","weights_repo":"flock-io/this-that-model-1.2","served_checkpoint":null},"completed":120340,"counts":{"ok":119898},"lifted":false,"calibration":{"acc":0.5882,"conf":0.7865,"ece":0.1983,"brier":0.5907,"over95":0.0785,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0119,"conf":0.0848,"acc":0.0291},{"lo":0.1,"w":0.0334,"conf":0.1451,"acc":0.0451},{"lo":0.2,"w":0.018,"conf":0.2502,"acc":0.1757},{"lo":0.3,"w":0.027,"conf":0.3546,"acc":0.1461},{"lo":0.4,"w":0.0385,"conf":0.4552,"acc":0.3002},{"lo":0.5,"w":0.0951,"conf":0.5488,"acc":0.4085},{"lo":0.6,"w":0.0846,"conf":0.649,"acc":0.4596},{"lo":0.7,"w":0.0916,"conf":0.7513,"acc":0.5311},{"lo":0.8,"w":0.1132,"conf":0.8547,"acc":0.5823},{"lo":0.9,"w":0.4868,"conf":0.9786,"acc":0.771}],"areas":{"knowledge":{"ece":0.2136,"acc":0.5171,"conf":0.7307},"language":{"ece":0.1613,"acc":0.7028,"conf":0.8627},"retrieval":{"ece":0.2394,"acc":0.6524,"conf":0.8918},"tools":{"ece":0.1689,"acc":0.6815,"conf":0.8496},"arts":{"ece":0.2192,"acc":0.4308,"conf":0.65}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":44.3,"p95":57.2,"mean":92.5},"results":{"1":{"score":0.8235,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":47.3},"2":{"score":0.5401,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":185.3},"3":{"score":0.5236,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7157,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":47.6},"5":{"score":0.6608,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":52.1},"6":{"score":0.6848,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":53.7},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5428,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":47.8},"11":{"score":0.6468,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.4966,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":43.4},"20":{"score":0.5526,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":43.3},"21":{"score":0.6043,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":43.0},"22":{"score":0.0135,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":109.8},"23":{"score":0.5808,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":44.7},"24":{"score":0.5958,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":43.2},"25":{"score":0.3316,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9213,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":43.1},"27":{"score":0.8319,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":43.1},"28":{"score":0.674,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":43.0},"29":{"score":0.8067,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":43.7},"30":{"score":0.2817,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":43.4},"31":{"score":0.0868,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":46.4},"32":{"score":0.5465,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3127,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":47.5},"36":{"score":0.1951,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3783,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":44.5},"38":{"score":0.0728,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":144.0},"39":{"score":0.747,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.1942,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":43.2,"tracks":[{"track":"A · Arabic","score":0.0372,"headline":false},{"track":"A · English","score":0.1942,"headline":true},{"track":"C · Arabic pairs","score":0.33,"headline":false},{"track":"C · English pairs","score":0.51,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4996,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":43.4},"42":{"score":0.6757,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":46.0},"43":{"score":0.3193,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5642,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":43.3},"45":{"score":0.1238,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.3267,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":46.8},"50":{"score":0.4057,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":47.4},"56":{"score":0.5,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":47.0,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.3718,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":43.6,"edition_added":"v0.2","jev":0.827},"58":{"score":0.4868,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":43.6,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.5709,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":46.3,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.4356,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.1417,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":47.6,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.597,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":45.4,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":44.86,"balanced_skill":28.14,"breadth_skill":26.37},"frozen_scores":{"balanced_raw":33.45,"balanced_skill":15.85,"breadth_skill":13.63},"categories":[{"id":"knowledge","raw":0.3415,"skill":0.1514,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.5048,"skill":0.3191,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.5093,"skill":0.3283,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.5051,"skill":0.4497,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.3557,"skill":0.1183,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.8235,"skill":0.7617,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.5401,"skill":0.4689,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.6848,"skill":0.2606,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.6501,"headline":false},{"track":"RouterBench-5shot","score":0.7196,"headline":false}]},"11":{"raw":0.6468,"skill":0.4892,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.5526,"skill":0.1052,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6043,"skill":0.2085,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0126,"skill":0.0049,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5505,"skill":0.101,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.617,"skill":0.4893,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3316,"skill":0.1088,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.2817,"skill":0.129,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.3442,"headline":false},{"track":"GSM8K-10choice","score":0.2191,"headline":false}]},"31":{"raw":0.0868,"skill":0.0054,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.1951,"skill":0.0895,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.3783,"skill":0.2201,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.1942,"skill":0.0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0372,"headline":false},{"track":"A · English","score":0.1942,"headline":true},{"track":"C · Arabic pairs","score":0.33,"headline":false},{"track":"C · English pairs","score":0.51,"headline":false}]},"41":{"raw":0.4996,"skill":0.2494,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3193,"skill":0.0,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5642,"skill":0.1284,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4057,"skill":0.1375,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.3718,"skill":0.2934,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.4868,"skill":0.2561,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.5709,"skill":0.4279,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.4356,"skill":0.2945,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.1417,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.597,"skill":0.194,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5465,"skill":0.279,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.3127,"skill":0.3036,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1238,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.4966,"skill":0.246,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.674,"skill":0.348,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.8067,"skill":0.7423,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0728,"skill":0.0431,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.747,"skill":0.6279,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.6757,"skill":0.3691,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.7157,"skill":0.712,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.6608,"skill":0.6588,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.5236,"skill":0.5144,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"TypedDecider.decide one forward per row, bf16, fla chunk_gated_delta_rule + causal-conv1d 1.7.0 CUDA kernel (board used the torch conv fallback), sdpa full attention, torch 2.14.0+cu130"},{"engine":"verdict-8af2496e","name":"Verdict","long_name":"Verdict","variant":null,"meta":{"kind":"full fine-tune","technique":"GLi family","technique_name":"GLiClass","served_params":151378176,"params_estimated":false,"params_from_name":false,"trained_bytes":605529340,"model_url":"https://huggingface.co/heman10x/rlcd-modernbert-151m","code_url":"https://github.com/Heman10x-NGU/Verdict-open-jev","base_model":"knowledgator/gliclass-modern-base-v2.0","base_url":"https://huggingface.co/knowledgator/gliclass-modern-base-v2.0","weights_repo":"heman10x/rlcd-modernbert-151m","served_checkpoint":null},"completed":120340,"counts":{"ok":49239,"abstained":56045,"unsupported":15056},"lifted":false,"calibration":{"acc":0.3689,"conf":0.5228,"ece":0.154,"brier":0.6841,"over95":0,"n":58189,"benchmarks":28,"rel":[{"lo":0.0,"w":0.0151,"conf":0.0763,"acc":0.0637},{"lo":0.1,"w":0.0148,"conf":0.1474,"acc":0.1513},{"lo":0.2,"w":0.0319,"conf":0.2639,"acc":0.1987},{"lo":0.3,"w":0.1264,"conf":0.3563,"acc":0.2337},{"lo":0.4,"w":0.1942,"conf":0.4526,"acc":0.3511},{"lo":0.5,"w":0.2984,"conf":0.5445,"acc":0.3913},{"lo":0.6,"w":0.2431,"conf":0.6472,"acc":0.4543},{"lo":0.7,"w":0.0557,"conf":0.7378,"acc":0.453},{"lo":0.8,"w":0.0191,"conf":0.8387,"acc":0.4591},{"lo":0.9,"w":0.0013,"conf":0.9127,"acc":0.332}],"areas":{"knowledge":{"ece":0.1816,"acc":0.3151,"conf":0.4964},"language":{"ece":0.0844,"acc":0.4755,"conf":0.5469},"retrieval":{"ece":0.3057,"acc":0.1527,"conf":0.4584},"tools":{"ece":0.3816,"acc":0.241,"conf":0.6226},"arts":{"ece":0.0817,"acc":0.4493,"conf":0.5238}}},"gaps":{"share":0.5908,"reasons":[{"reason":"chose its own abstain option instead of an answer","share":0.4657},{"reason":"supports only 2–24 answer options per question","share":0.1224},{"reason":"prompt longer than its 8,192-token context window","share":0.0027}]},"latency":{"median":11.4,"p95":60.1,"mean":21.0},"results":{"1":{"score":0.3878,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":17.6},"2":{"score":0.1904,"jev":0.6621,"metric":"nDCG@10","requests":685,"answered":496,"unsupported":5,"errors":0,"median_ms":null},"3":{"score":null,"jev":null,"metric":null,"requests":508,"answered":0,"unsupported":508,"errors":0,"median_ms":null},"4":{"score":null,"jev":null,"metric":null,"requests":3080,"answered":0,"unsupported":3080,"errors":0,"median_ms":null},"5":{"score":null,"jev":null,"metric":null,"requests":5500,"answered":0,"unsupported":5500,"errors":0,"median_ms":null},"6":{"score":0.5084,"jev":0.7566,"metric":"selected quality (quality objective)","requests":10000,"answered":7536,"unsupported":0,"errors":0,"median_ms":43.6},"9":{"score":null,"jev":null,"metric":null,"requests":88,"answered":0,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.0792,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":18.4},"11":{"score":0.2116,"jev":0.7251,"metric":"macro-F1","requests":123,"answered":31,"unsupported":1,"errors":0,"median_ms":null},"12":{"score":0.1492,"jev":0.7416,"metric":"macro-F1","requests":3200,"answered":328,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.4761,"jev":0.9046,"metric":"accuracy","requests":5000,"answered":2283,"unsupported":0,"errors":0,"median_ms":10.7},"21":{"score":0.386,"jev":0.6228,"metric":"accuracy","requests":2628,"answered":228,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":null,"jev":null,"metric":null,"requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"23":{"score":0.6562,"jev":0.6979,"metric":"accuracy","requests":5000,"answered":96,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.3088,"jev":0.9094,"metric":"accuracy","requests":14033,"answered":4812,"unsupported":0,"errors":0,"median_ms":10.6},"25":{"score":0.2874,"jev":0.7816,"metric":"accuracy","requests":196,"answered":87,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.5206,"jev":0.9905,"metric":"accuracy","requests":2376,"answered":632,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.2738,"jev":0.9911,"metric":"accuracy","requests":1172,"answered":336,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.494,"jev":0.9186,"metric":"accuracy","requests":1267,"answered":1241,"unsupported":0,"errors":0,"median_ms":9.5},"29":{"score":0.3506,"jev":0.9561,"metric":"accuracy","requests":10042,"answered":6896,"unsupported":0,"errors":0,"median_ms":11.2},"30":{"score":0.2333,"jev":0.8067,"metric":"accuracy","requests":2638,"answered":474,"unsupported":0,"errors":0,"median_ms":10.9},"31":{"score":0.1735,"jev":0.2913,"metric":"accuracy","requests":5000,"answered":1349,"unsupported":3641,"errors":0,"median_ms":11.8},"32":{"score":0.4101,"jev":0.6646,"metric":"accuracy","requests":752,"answered":656,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0119,"jev":0.2143,"metric":"case exact accuracy","requests":1650,"answered":84,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.0903,"jev":0.4486,"metric":"nDCG@10","requests":220,"answered":61,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.1277,"jev":0.5555,"metric":"macro-F1","requests":5000,"answered":2146,"unsupported":0,"errors":0,"median_ms":11.5},"38":{"score":0.0365,"jev":0.2957,"metric":"per-review F1","requests":1565,"answered":1559,"unsupported":0,"errors":0,"median_ms":80.6},"39":{"score":0.4211,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.1012,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":3614,"unsupported":0,"errors":0,"median_ms":9.5,"tracks":[{"track":"A · Arabic","score":0.0,"headline":false},{"track":"A · English","score":0.1012,"headline":true},{"track":"C · Arabic pairs","score":0.195,"headline":false},{"track":"C · English pairs","score":0.1,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3337,"jev":0.6661,"metric":"macro-F1","requests":3006,"answered":1388,"unsupported":0,"errors":0,"median_ms":10.9},"42":{"score":0.5155,"jev":0.8627,"metric":"macro-F1","requests":5500,"answered":891,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.3926,"jev":0.7375,"metric":"accuracy","requests":570,"answered":461,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.3788,"jev":0.7879,"metric":"accuracy","requests":5000,"answered":66,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1496,"jev":0.2044,"metric":"accuracy","requests":501,"answered":274,"unsupported":4,"errors":0,"median_ms":null},"48":{"score":0.227,"jev":0.1651,"metric":"Brier (lower is better)","requests":10139,"answered":5067,"unsupported":0,"errors":0,"median_ms":15.4},"50":{"score":0.3769,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1661,"unsupported":0,"errors":0,"median_ms":16.2},"56":{"score":0.5532,"metric":"accuracy","requests":2000,"answered":47,"unsupported":0,"errors":1953,"median_ms":null,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.1345,"metric":"accuracy","requests":12032,"answered":7116,"unsupported":0,"errors":4916,"median_ms":10.9,"edition_added":"v0.2","jev":0.827},"58":{"score":0.3361,"metric":"accuracy","requests":5507,"answered":5444,"unsupported":0,"errors":63,"median_ms":10.7,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.3681,"metric":"accuracy","requests":3652,"answered":3540,"unsupported":0,"errors":112,"median_ms":13.6,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.8,"metric":"accuracy","requests":528,"answered":5,"unsupported":0,"errors":523,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.119,"metric":"F1 on hallucinated class","requests":2700,"answered":2666,"unsupported":0,"errors":34,"median_ms":14.9,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5173,"metric":"accuracy","requests":4000,"answered":1562,"unsupported":0,"errors":2438,"median_ms":11.6,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":12.94,"balanced_skill":1.87,"breadth_skill":1.69},"frozen_scores":{"balanced_raw":9.17,"balanced_skill":1.08,"breadth_skill":1.0},"categories":[{"id":"knowledge","raw":0.1376,"skill":0.0036,"coverage":0.4775,"pending":0,"n":10},{"id":"language","raw":0.1737,"skill":0.0145,"coverage":0.6384,"pending":0,"n":10},{"id":"retrieval","raw":0.0431,"skill":0.0,"coverage":0.1782,"pending":0,"n":6},{"id":"tools","raw":0.1778,"skill":0.0658,"coverage":0.5358,"pending":0,"n":5},{"id":"arts","raw":0.0781,"skill":0.0203,"coverage":0.2805,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.3878,"skill":0.1736,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.1379,"skill":0.0044,"coverage":0.7241,"random":0.1341,"tracks":[]},"6":{"raw":0.3006,"skill":0.0,"coverage":0.591,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.2851,"headline":false},{"track":"RouterBench-5shot","score":0.316,"headline":false}]},"11":{"raw":0.0631,"skill":0.0,"coverage":0.252,"random":0.3085,"tracks":[]},"20":{"raw":0.0858,"skill":0.0,"coverage":0.3132,"random":0.5,"tracks":[]},"21":{"raw":0.0335,"skill":0.0,"coverage":0.0868,"random":0.5,"tracks":[]},"22":{"raw":0.0,"skill":0.0,"coverage":0,"random":0.0078,"tracks":[]},"23":{"raw":0.0069,"skill":0.0,"coverage":0.0192,"random":0.5,"tracks":[]},"24":{"raw":0.1001,"skill":0.0,"coverage":0.3429,"random":0.25,"tracks":[]},"25":{"raw":0.1276,"skill":0.0,"coverage":0.4439,"random":0.25,"tracks":[]},"30":{"raw":0.0133,"skill":0.0,"coverage":0.0569,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.0159,"headline":false},{"track":"GSM8K-10choice","score":0.0106,"headline":false}]},"31":{"raw":0.0468,"skill":0.0,"coverage":0.2698,"random":0.0819,"tracks":[]},"36":{"raw":0.025,"skill":0.0,"coverage":0.2773,"random":0.116,"tracks":[]},"37":{"raw":0.0393,"skill":0.0,"coverage":0.4292,"random":0.2027,"tracks":[]},"40":{"raw":0.1012,"skill":0.0,"coverage":0.7857,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0,"headline":false},{"track":"A · English","score":0.1012,"headline":true},{"track":"C · Arabic pairs","score":0.195,"headline":false},{"track":"C · English pairs","score":0.1,"headline":false}]},"41":{"raw":0.1301,"skill":0.0,"coverage":0.4617,"random":0.3333,"tracks":[]},"43":{"raw":0.3175,"skill":0.0,"coverage":0.8088,"random":0.3697,"tracks":[]},"44":{"raw":0.005,"skill":0.0,"coverage":0.0132,"random":0.5,"tracks":[]},"50":{"raw":0.3735,"skill":0.0908,"coverage":0.9911,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.013,"skill":0.0,"coverage":0.0235,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.0795,"skill":0.0,"coverage":0.5914,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.3323,"skill":0.0322,"coverage":0.9886,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.3568,"skill":0.1424,"coverage":0.9693,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.0076,"skill":0.0,"coverage":0.0095,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.1175,"skill":0.0,"coverage":0.9874,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.202,"skill":0.0,"coverage":0.3905,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.3577,"skill":0.0,"coverage":0.8723,"random":0.371,"tracks":[]},"33":{"raw":0.0006,"skill":0.0,"coverage":0.0509,"random":0.0131,"tracks":[]},"45":{"raw":0.0818,"skill":0.0,"coverage":0.5469,"random":0.1641,"tracks":[]},"12":{"raw":0.0153,"skill":0.0,"coverage":0.1025,"random":0.3324,"tracks":[]},"28":{"raw":0.4839,"skill":0.0,"coverage":0.9795,"random":0.5,"tracks":[]},"29":{"raw":0.2408,"skill":0.0,"coverage":0.6867,"random":0.25,"tracks":[]},"38":{"raw":0.0364,"skill":0.0055,"coverage":0.9962,"random":0.031,"tracks":[]},"39":{"raw":0.4211,"skill":0.1486,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.0835,"skill":0.0,"coverage":0.162,"random":0.486,"tracks":[]},"4":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0127,"tracks":[]},"5":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.006,"tracks":[]},"3":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0,"tracks":[]},"48":{"raw":0.046,"skill":0.046,"coverage":0.4998,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.3069,"pending":0.24,"latency_rows":357,"latency_path":"author DecisionEngine (ModernBERT 151M) evaluate, max_length 8192 lift as board; venv-vl torch 2.13.0+cu130 from the rerun freeze. Native abstentions are not timed (357 of 750 rows answered)"},{"engine":"vllm-pr57250","name":"mmastrac diffusiongemma (vLLM PR 57250)","long_name":"mmastrac diffgemma","variant":null,"meta":{"kind":"inference technique","technique":"diffusion","technique_name":"diffusion","served_params":25823778864,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/vllm-project/vllm/pull/57250","base_model":"google/diffusiongemma-26B-A4B-it","base_url":"https://huggingface.co/google/diffusiongemma-26B-A4B-it","weights_repo":null,"served_checkpoint":"google/diffusiongemma-26B-A4B-it"},"completed":120340,"counts":{"ok":105818,"unsupported":14522},"lifted":false,"calibration":{"acc":0.6495,"conf":0.8513,"ece":0.2019,"brier":0.535,"over95":0.0907,"n":70144,"benchmarks":28,"rel":[{"lo":0.0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0006,"conf":0.1821,"acc":0.2358},{"lo":0.2,"w":0.008,"conf":0.2643,"acc":0.0903},{"lo":0.3,"w":0.0199,"conf":0.3628,"acc":0.2245},{"lo":0.4,"w":0.0327,"conf":0.459,"acc":0.2679},{"lo":0.5,"w":0.0781,"conf":0.5493,"acc":0.3685},{"lo":0.6,"w":0.0803,"conf":0.6521,"acc":0.4727},{"lo":0.7,"w":0.0839,"conf":0.7504,"acc":0.5228},{"lo":0.8,"w":0.1121,"conf":0.8526,"acc":0.5651},{"lo":0.9,"w":0.5845,"conf":0.9804,"acc":0.7895}],"areas":{"knowledge":{"ece":0.2351,"acc":0.5659,"conf":0.8009},"language":{"ece":0.1587,"acc":0.7472,"conf":0.9028},"retrieval":{"ece":0.2883,"acc":0.5026,"conf":0.7909},"tools":{"ece":0.0763,"acc":0.8329,"conf":0.9092},"arts":{"ece":0.2341,"acc":0.6413,"conf":0.8755}}},"gaps":{"share":0.1207,"reasons":[{"reason":"supports at most 26 answer options per question","share":0.1205},{"reason":"structured_server: question 'q_78d37c943d': label 'C' is not a single token","share":0.0},{"reason":"structured_server: question 'q_c3e60b168d': label 'C' is not a single token","share":0.0}]},"latency":{"median":126.7,"p95":274.6,"mean":162.8},"results":{"1":{"score":0.9622,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":132.0},"2":{"score":0.4631,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":664.5},"3":{"score":null,"jev":null,"metric":null,"requests":508,"answered":0,"unsupported":508,"errors":0,"median_ms":null},"4":{"score":null,"jev":null,"metric":null,"requests":3080,"answered":0,"unsupported":3080,"errors":0,"median_ms":null},"5":{"score":null,"jev":null,"metric":null,"requests":5500,"answered":0,"unsupported":5500,"errors":0,"median_ms":null},"6":{"score":0.7988,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":251.1},"9":{"score":0,"jev":0.5455,"metric":"case exact accuracy","requests":88,"answered":66,"unsupported":22,"errors":0,"median_ms":null},"10":{"score":0.4299,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":96.3},"11":{"score":0.5632,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5574,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":114.8},"20":{"score":0.8854,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":123.0},"21":{"score":0.6084,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":117.3},"22":{"score":null,"jev":null,"metric":null,"requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"23":{"score":0.6098,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":141.0},"24":{"score":0.7862,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":114.5},"25":{"score":0.4898,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9811,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":50.9},"27":{"score":0.9403,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":50.1},"28":{"score":0.708,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":116.4},"29":{"score":0.8173,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":58.5},"30":{"score":0.3639,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":126.2},"31":{"score":0.2173,"jev":0.2746,"metric":"accuracy","requests":5000,"answered":1588,"unsupported":3412,"errors":0,"median_ms":142.9},"32":{"score":0.625,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2564,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":176.5},"36":{"score":0.2259,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5339,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":132.1},"38":{"score":0.1443,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":728.1},"39":{"score":0.8979,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5895,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":119.4,"tracks":[{"track":"A · Arabic","score":0.4853,"headline":false},{"track":"A · English","score":0.5895,"headline":true},{"track":"C · Arabic pairs","score":0.815,"headline":false},{"track":"C · English pairs","score":0.955,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.6086,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":118.2},"42":{"score":0.7446,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":139.5},"43":{"score":0.4526,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6356,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":116.1},"45":{"score":0.1038,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2141,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":148.7},"50":{"score":0.4666,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":163.0},"56":{"score":0.824,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":212.5,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.413,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":128.9,"edition_added":"v0.2","jev":0.827},"58":{"score":0.6644,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":122.1,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.4874,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":118.6,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.5739,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.6789,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":154.3,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.6913,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":134.7,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":45.72,"balanced_skill":32.24,"breadth_skill":31.41},"frozen_scores":{"balanced_raw":40.96,"balanced_skill":27.48,"breadth_skill":23.09},"categories":[{"id":"knowledge","raw":0.4081,"skill":0.249,"coverage":0.9368,"pending":0,"n":10},{"id":"language","raw":0.6346,"skill":0.4593,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.3516,"skill":0.2417,"coverage":0.6364,"pending":0,"n":6},{"id":"tools","raw":0.3898,"skill":0.3399,"coverage":0.7315,"pending":0,"n":5},{"id":"arts","raw":0.4607,"skill":0.2881,"coverage":0.8611,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9622,"skill":0.949,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.4631,"skill":0.3799,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7988,"skill":0.5256,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7852,"headline":false},{"track":"RouterBench-5shot","score":0.8123,"headline":false}]},"11":{"raw":0.5632,"skill":0.3683,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.8896,"skill":0.7791,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6084,"skill":0.2169,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0,"skill":0.0,"coverage":0,"random":0.0078,"tracks":[]},"23":{"raw":0.6063,"skill":0.2125,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7911,"skill":0.7215,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4898,"skill":0.3197,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.3639,"skill":0.2284,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.4162,"headline":false},{"track":"GSM8K-10choice","score":0.3116,"headline":false}]},"31":{"raw":0.069,"skill":0.0,"coverage":0.3176,"random":0.0819,"tracks":[]},"36":{"raw":0.2259,"skill":0.1243,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.5339,"skill":0.4154,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.5895,"skill":0.4719,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4853,"headline":false},{"track":"A · English","score":0.5895,"headline":true},{"track":"C · Arabic pairs","score":0.815,"headline":false},{"track":"C · English pairs","score":0.955,"headline":false}]},"41":{"raw":0.6086,"skill":0.4128,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4526,"skill":0.1315,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6356,"skill":0.2712,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4666,"skill":0.2258,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.824,"skill":0.648,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.413,"skill":0.3398,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6644,"skill":0.5136,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.4874,"skill":0.3165,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.5739,"skill":0.4674,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.6789,"skill":0.3342,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.6913,"skill":0.3826,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.625,"skill":0.4038,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2564,"skill":0.2465,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1038,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.5574,"skill":0.337,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.708,"skill":0.416,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.8173,"skill":0.7564,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.1443,"skill":0.1169,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8979,"skill":0.8498,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7446,"skill":0.5031,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0127,"tracks":[]},"5":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.006,"tracks":[]},"3":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":0.75,"random":0.0,"tracks":[]},"48":{"raw":0.1436,"skill":0.1436,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.6927,"pending":0.24,"latency_rows":678,"latency_path":"structured_server.py over vLLM PR 57250 nightly, prefix caching, default attention backend, one sequential HTTP client","new_in_edition":true},{"engine":"winnow-12b-q8","name":"Winnow-12B","long_name":"Winnow-12B","variant":"Q8_0","meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":11959730224,"params_estimated":false,"params_from_name":false,"trained_bytes":12669646592,"model_url":"https://huggingface.co/EldanRing/Winnow-12B","code_url":"https://github.com/EldanRing/winnow-inference","base_model":"google/gemma-4-12B","base_url":"https://huggingface.co/google/gemma-4-12B","weights_repo":"EldanRing/Winnow-12B","served_checkpoint":"EldanRing/Winnow-12B"},"completed":120340,"counts":{"ok":120340},"lifted":true,"calibration":{"acc":0.6697,"conf":0.8376,"ece":0.1679,"brier":0.4862,"over95":0.0624,"n":72594,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0035,"conf":0.0806,"acc":0.0},{"lo":0.1,"w":0.0152,"conf":0.1522,"acc":0.1179},{"lo":0.2,"w":0.0227,"conf":0.2538,"acc":0.1849},{"lo":0.3,"w":0.0276,"conf":0.3543,"acc":0.2116},{"lo":0.4,"w":0.0335,"conf":0.4537,"acc":0.2994},{"lo":0.5,"w":0.0578,"conf":0.5508,"acc":0.435},{"lo":0.6,"w":0.054,"conf":0.6514,"acc":0.4727},{"lo":0.7,"w":0.076,"conf":0.7509,"acc":0.4814},{"lo":0.8,"w":0.1041,"conf":0.8558,"acc":0.553},{"lo":0.9,"w":0.6056,"conf":0.976,"acc":0.8306}],"areas":{"knowledge":{"ece":0.1675,"acc":0.6121,"conf":0.7783},"language":{"ece":0.1134,"acc":0.7995,"conf":0.9129},"retrieval":{"ece":0.3164,"acc":0.6017,"conf":0.918},"tools":{"ece":0.1021,"acc":0.8499,"conf":0.9437},"arts":{"ece":0.1953,"acc":0.5092,"conf":0.7045}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":72.5,"p95":356.2,"mean":134.4},"results":{"1":{"score":0.9475,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":125.2},"2":{"score":0.6519,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1053.8},"3":{"score":0.6732,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7511,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":173.1},"5":{"score":0.8695,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":233.6},"6":{"score":0.7976,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":290.9},"9":{"score":0.5455,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.1189,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":145.9},"11":{"score":0.7289,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6674,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":52.5},"20":{"score":0.8314,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":58.0},"21":{"score":0.6126,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":48.6},"22":{"score":0.081,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":667.1},"23":{"score":0.6018,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":84.1},"24":{"score":0.7732,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":48.6},"25":{"score":0.3878,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9798,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":49.4},"27":{"score":0.948,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":28.3},"28":{"score":0.7411,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":47.1},"29":{"score":0.8512,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":54.5},"30":{"score":0.539,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":63.3},"31":{"score":0.1734,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":132.5},"32":{"score":0.6104,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3358,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":164.3},"36":{"score":0.4528,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5304,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":76.0},"38":{"score":0.1546,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":985.8},"39":{"score":0.9174,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5773,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":48.2,"tracks":[{"track":"A · Arabic","score":0.4759,"headline":false},{"track":"A · English","score":0.5773,"headline":true},{"track":"C · Arabic pairs","score":0.84,"headline":false},{"track":"C · English pairs","score":0.96,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.6863,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":53.2},"42":{"score":0.8294,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":91.2},"43":{"score":0.6947,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6856,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":55.6},"45":{"score":0.1038,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.1926,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":113.0},"50":{"score":0.4183,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":128.3},"56":{"score":0.659,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":148.9,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.5483,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":61.4,"edition_added":"v0.2","jev":0.827},"58":{"score":0.7129,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":62.5,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.8324,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":108.6,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.6477,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.7708,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":116.5,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.7205,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":82.5,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":61.91,"balanced_skill":50.02,"breadth_skill":48.3},"frozen_scores":{"balanced_raw":45.27,"balanced_skill":32.9,"breadth_skill":27.69},"categories":[{"id":"knowledge","raw":0.4761,"skill":0.3382,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.6959,"skill":0.5599,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.6664,"skill":0.5405,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.736,"skill":0.7102,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4818,"skill":0.2997,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9475,"skill":0.9291,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6519,"skill":0.598,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7976,"skill":0.5225,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7855,"headline":false},{"track":"RouterBench-5shot","score":0.8097,"headline":false}]},"11":{"raw":0.7289,"skill":0.6079,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.8342,"skill":0.6684,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6126,"skill":0.2253,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0804,"skill":0.0732,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.6002,"skill":0.2004,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.782,"skill":0.7094,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3878,"skill":0.1837,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.539,"skill":0.4471,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.6338,"headline":false},{"track":"GSM8K-10choice","score":0.4443,"headline":false}]},"31":{"raw":0.1734,"skill":0.0997,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4528,"skill":0.381,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.5304,"skill":0.411,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.5773,"skill":0.4562,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4759,"headline":false},{"track":"A · English","score":0.5773,"headline":true},{"track":"C · Arabic pairs","score":0.84,"headline":false},{"track":"C · English pairs","score":0.96,"headline":false}]},"41":{"raw":0.6863,"skill":0.5295,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.6947,"skill":0.5157,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6856,"skill":0.3712,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4183,"skill":0.1557,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.659,"skill":0.318,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.5483,"skill":0.492,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.7129,"skill":0.5839,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.8324,"skill":0.7765,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.6477,"skill":0.5596,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.7708,"skill":0.5248,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.7205,"skill":0.441,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.6104,"skill":0.3806,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.3358,"skill":0.327,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1038,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.6674,"skill":0.5018,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.7411,"skill":0.4822,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.8512,"skill":0.8016,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.1546,"skill":0.1276,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.9174,"skill":0.8785,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.8294,"skill":0.6681,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.7511,"skill":0.7479,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.8695,"skill":0.8687,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.6732,"skill":0.6669,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.5455,"skill":0.5455,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.2296,"skill":0.2296,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"winnow-inference llama.cpp CUDA server (6c2b3c0 + 255-label lift, WINNOW_LABELS_LIFT=1), Winnow-12B Q8_0 GGUF fully on GPU, n_ctx 131072, batch 2048 / ubatch 1024, one slot, one sequential HTTP client; same build and settings as the board","new_in_edition":true},{"engine":"winnow-e4b-q8","name":"Winnow-E4B","long_name":"Winnow-E4B","variant":"Q8_0","meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":7996156490,"params_estimated":false,"params_from_name":false,"trained_bytes":8005437472,"model_url":"https://huggingface.co/EldanRing/Winnow-E4B","code_url":"https://github.com/EldanRing/winnow-inference","base_model":"google/gemma-4-E4B","base_url":"https://huggingface.co/google/gemma-4-E4B","weights_repo":"EldanRing/Winnow-E4B","served_checkpoint":"EldanRing/Winnow-E4B"},"completed":120340,"counts":{"ok":119898},"lifted":false,"calibration":{"acc":0.6357,"conf":0.6933,"ece":0.058,"brier":0.4613,"over95":0.0069,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0284,"conf":0.063,"acc":0.0211},{"lo":0.1,"w":0.0309,"conf":0.1529,"acc":0.132},{"lo":0.2,"w":0.0331,"conf":0.2598,"acc":0.2652},{"lo":0.3,"w":0.0523,"conf":0.3491,"acc":0.2461},{"lo":0.4,"w":0.0502,"conf":0.4511,"acc":0.3741},{"lo":0.5,"w":0.1308,"conf":0.5503,"acc":0.4842},{"lo":0.6,"w":0.1353,"conf":0.6511,"acc":0.5491},{"lo":0.7,"w":0.1184,"conf":0.7494,"acc":0.6597},{"lo":0.8,"w":0.137,"conf":0.8518,"acc":0.799},{"lo":0.9,"w":0.2836,"conf":0.9583,"acc":0.9358}],"areas":{"knowledge":{"ece":0.0838,"acc":0.5408,"conf":0.6246},"language":{"ece":0.0325,"acc":0.7688,"conf":0.7998},"retrieval":{"ece":0.1093,"acc":0.6314,"conf":0.7336},"tools":{"ece":0.0471,"acc":0.8372,"conf":0.8238},"arts":{"ece":0.0567,"acc":0.4874,"conf":0.5423}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":45.0,"p95":188.3,"mean":75.6},"results":{"1":{"score":0.9144,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":75.3},"2":{"score":0.603,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":560.1},"3":{"score":0.7303,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.712,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":95.3},"5":{"score":0.826,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":120.0},"6":{"score":0.7983,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":149.1},"9":{"score":0.1023,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.3729,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":81.4},"11":{"score":0.7334,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5617,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":34.8},"20":{"score":0.8152,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":35.6},"21":{"score":0.6027,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":32.3},"22":{"score":0.036,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":328.6},"23":{"score":0.5662,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":51.0},"24":{"score":0.6701,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":32.5},"25":{"score":0.3622,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.96,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":33.2},"27":{"score":0.8857,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":18.9},"28":{"score":0.6606,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":31.8},"29":{"score":0.7662,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":34.7},"30":{"score":0.3161,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":39.7},"31":{"score":0.15,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":69.4},"32":{"score":0.5519,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2921,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":94.9},"36":{"score":0.4155,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4655,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":48.1},"38":{"score":0.2056,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":552.3},"39":{"score":0.8853,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5383,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":32.3,"tracks":[{"track":"A · Arabic","score":0.3571,"headline":false},{"track":"A · English","score":0.5383,"headline":true},{"track":"C · Arabic pairs","score":0.625,"headline":false},{"track":"C · English pairs","score":0.945,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.631,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":35.7},"42":{"score":0.7391,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":54.2},"43":{"score":0.5018,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6032,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":34.8},"45":{"score":0.1257,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2451,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":65.6},"50":{"score":0.4421,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":74.6},"56":{"score":0.5095,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":92.7,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.4377,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":39.6,"edition_added":"v0.2","jev":0.827},"58":{"score":0.6287,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":38.8,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.8502,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":66.0,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.5568,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.6081,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":66.2,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.6478,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":49.7,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":54.84,"balanced_skill":39.89,"breadth_skill":37.73},"frozen_scores":{"balanced_raw":40.95,"balanced_skill":26.64,"breadth_skill":22.67},"categories":[{"id":"knowledge","raw":0.3963,"skill":0.2229,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.6346,"skill":0.4511,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.6011,"skill":0.438,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.6535,"skill":0.6251,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4211,"skill":0.2277,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9144,"skill":0.8844,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.603,"skill":0.5415,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7983,"skill":0.5244,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7849,"headline":false},{"track":"RouterBench-5shot","score":0.8117,"headline":false}]},"11":{"raw":0.7334,"skill":0.6145,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.8184,"skill":0.6368,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6027,"skill":0.2055,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0321,"skill":0.0245,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5564,"skill":0.1129,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6759,"skill":0.5679,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3622,"skill":0.1497,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.3161,"skill":0.1718,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.3844,"headline":false},{"track":"GSM8K-10choice","score":0.2479,"headline":false}]},"31":{"raw":0.15,"skill":0.0742,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4155,"skill":0.3388,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.4655,"skill":0.3296,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.5383,"skill":0.406,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3571,"headline":false},{"track":"A · English","score":0.5383,"headline":true},{"track":"C · Arabic pairs","score":0.625,"headline":false},{"track":"C · English pairs","score":0.945,"headline":false}]},"41":{"raw":0.631,"skill":0.4465,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.5018,"skill":0.2095,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6032,"skill":0.2064,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4421,"skill":0.1903,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5095,"skill":0.019,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.4377,"skill":0.3676,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6287,"skill":0.4618,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.8502,"skill":0.8003,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.5568,"skill":0.446,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.6081,"skill":0.1874,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.6478,"skill":0.2956,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5519,"skill":0.2876,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2921,"skill":0.2827,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1257,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.5617,"skill":0.3435,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.6606,"skill":0.3212,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.7662,"skill":0.6883,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.2056,"skill":0.1802,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8853,"skill":0.8313,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7391,"skill":0.4924,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.712,"skill":0.7083,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.826,"skill":0.8249,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.7303,"skill":0.7251,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.1023,"skill":0.1023,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0196,"skill":0.0196,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"winnow-inference llama.cpp CUDA server (77d1458 labels-lift build), Winnow-E4B Q8_0 GGUF fully on GPU, n_ctx 131072, batch 2048 / ubatch 1024, one server and one sequential HTTP client (board: 4 servers per card)","new_in_edition":true},{"engine":"xor","name":"Xor","long_name":"Xor Qwen3.6-35B-A3B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":35951822704,"params_estimated":false,"params_from_name":false,"trained_bytes":70214495944,"model_url":"https://huggingface.co/juspay/xor","code_url":null,"base_model":"Qwen/Qwen3.6-35B-A3B","base_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","weights_repo":"juspay/xor","served_checkpoint":null},"completed":120340,"counts":{"ok":105398,"unsupported":14500},"lifted":false,"calibration":{"acc":0.7093,"conf":0.7186,"ece":0.0149,"brier":0.3881,"over95":0.0041,"n":32693,"benchmarks":28,"rel":[{"lo":0.0,"w":0.0001,"conf":0.0999,"acc":0.0},{"lo":0.1,"w":0.0105,"conf":0.1611,"acc":0.1752},{"lo":0.2,"w":0.0202,"conf":0.25,"acc":0.243},{"lo":0.3,"w":0.0465,"conf":0.3535,"acc":0.2913},{"lo":0.4,"w":0.0642,"conf":0.4506,"acc":0.3605},{"lo":0.5,"w":0.1549,"conf":0.5488,"acc":0.53},{"lo":0.6,"w":0.1471,"conf":0.6513,"acc":0.6501},{"lo":0.7,"w":0.1426,"conf":0.7497,"acc":0.7482},{"lo":0.8,"w":0.1517,"conf":0.8494,"acc":0.8602},{"lo":0.9,"w":0.2622,"conf":0.9533,"acc":0.9572}],"areas":{"knowledge":{"ece":0.0355,"acc":0.6403,"conf":0.6751},"language":{"ece":0.0565,"acc":0.8153,"conf":0.7608},"retrieval":{"ece":0.2824,"acc":0.4149,"conf":0.6973},"tools":{"ece":0.0637,"acc":0.9715,"conf":0.909},"arts":{"ece":0.0302,"acc":0.6767,"conf":0.6588}}},"gaps":{"share":0.1209,"reasons":[{"reason":"xor server 422: q1: 2..26 options, got 151","share":0.0459},{"reason":"xor server 422: q1: 2..26 options, got 77","share":0.0257},{"reason":"xor server 422: chord: 2..26 options, got 129","share":0.0167}]},"latency":{"median":139.9,"p95":419.9,"mean":184.2},"results":{"1":{"score":0.9575,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":209.9},"2":{"score":0.6364,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":710.0},"3":{"score":null,"jev":null,"metric":null,"requests":508,"answered":0,"unsupported":508,"errors":0,"median_ms":null},"4":{"score":null,"jev":null,"metric":null,"requests":3080,"answered":0,"unsupported":3080,"errors":0,"median_ms":null},"5":{"score":null,"jev":null,"metric":null,"requests":5500,"answered":0,"unsupported":5500,"errors":0,"median_ms":null},"6":{"score":0.8002,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":339.9},"9":{"score":0.5341,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.2381,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":209.8},"11":{"score":0.7429,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6851,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":129.9},"20":{"score":0.9662,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":129.8},"21":{"score":0.6374,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":119.9},"22":{"score":null,"jev":null,"metric":null,"requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"23":{"score":0.6304,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":159.9},"24":{"score":0.8405,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":129.9},"25":{"score":0.4898,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9903,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":119.9},"27":{"score":0.9608,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":120.0},"28":{"score":0.8082,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":119.9},"29":{"score":0.9364,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":139.9},"30":{"score":0.7191,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":129.9},"31":{"score":0.2286,"jev":0.2746,"metric":"accuracy","requests":5000,"answered":1588,"unsupported":3412,"errors":0,"median_ms":159.9},"32":{"score":0.621,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2467,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":314.9},"36":{"score":0.4233,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4464,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":159.9},"38":{"score":0.1728,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":810.0},"39":{"score":0.8874,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5553,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":119.8,"tracks":[{"track":"A · Arabic","score":0.3082,"headline":false},{"track":"A · English","score":0.5553,"headline":true},{"track":"C · Arabic pairs","score":0.7,"headline":false},{"track":"C · English pairs","score":0.965,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.6883,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":129.9},"42":{"score":0.7255,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":169.7},"43":{"score":0.6,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6368,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":129.9},"45":{"score":0.1357,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2011,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":169.9},"50":{"score":0.4576,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":179.9},"56":{"score":0.654,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":229.6,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.6261,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":139.6,"edition_added":"v0.2","jev":0.827},"58":{"score":0.7096,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":129.9,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.7993,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":189.8,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.714,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.7028,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":179.8,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.7422,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":159.9,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":53.34,"balanced_skill":41.48,"breadth_skill":39.78},"frozen_scores":{"balanced_raw":45.55,"balanced_skill":33.74,"breadth_skill":27.66},"categories":[{"id":"knowledge","raw":0.4861,"skill":0.3433,"coverage":0.9368,"pending":0,"n":10},{"id":"language","raw":0.6972,"skill":0.5558,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.3561,"skill":0.2296,"coverage":0.6364,"pending":0,"n":6},{"id":"tools","raw":0.5776,"skill":0.5514,"coverage":0.7778,"pending":0,"n":5},{"id":"arts","raw":0.5061,"skill":0.3563,"coverage":0.8611,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9575,"skill":0.9426,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6364,"skill":0.5801,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.8002,"skill":0.5288,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7872,"headline":false},{"track":"RouterBench-5shot","score":0.8132,"headline":false}]},"11":{"raw":0.7429,"skill":0.6282,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.9664,"skill":0.9327,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6374,"skill":0.2747,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0,"skill":0.0,"coverage":0,"random":0.0078,"tracks":[]},"23":{"raw":0.634,"skill":0.2679,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8468,"skill":0.7957,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4898,"skill":0.3197,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.7191,"skill":0.6602,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.7506,"headline":false},{"track":"GSM8K-10choice","score":0.6876,"headline":false}]},"31":{"raw":0.0726,"skill":0.0,"coverage":0.3176,"random":0.0819,"tracks":[]},"36":{"raw":0.4233,"skill":0.3476,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.4464,"skill":0.3056,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.5553,"skill":0.428,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3082,"headline":false},{"track":"A · English","score":0.5553,"headline":true},{"track":"C · Arabic pairs","score":0.7,"headline":false},{"track":"C · English pairs","score":0.965,"headline":false}]},"41":{"raw":0.6883,"skill":0.5324,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.6,"skill":0.3654,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6368,"skill":0.2736,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4576,"skill":0.2129,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.654,"skill":0.308,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.6261,"skill":0.5795,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.7096,"skill":0.5791,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.7993,"skill":0.7324,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.714,"skill":0.6425,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.7028,"skill":0.3838,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.7422,"skill":0.4844,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.621,"skill":0.3975,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2467,"skill":0.2367,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1357,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.6851,"skill":0.5283,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.8082,"skill":0.6164,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.9364,"skill":0.9152,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.1728,"skill":0.1463,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8874,"skill":0.8344,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7255,"skill":0.466,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0127,"tracks":[]},"5":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.006,"tracks":[]},"3":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":0.0189,"tracks":[]},"9":{"raw":0.5341,"skill":0.5341,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.1956,"skill":0.1956,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.6927,"pending":0.24,"latency_rows":678,"latency_path":"author xor server.py (raised body caps) over SGLang image 6bcaa47d, TP=1 on one RTX PRO 6000 HF Job, OPENJEV_CACHE=0 as board, one sequential client","new_in_edition":true},{"engine":"lev","name":"lev","long_name":"Interfaze lev Qwen3.5-4B LoRA","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":184601885,"model_url":"https://huggingface.co/interfaze-ai/lev","code_url":"https://github.com/Abhinavexists/lev","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"interfaze-ai/lev","served_checkpoint":null},"completed":120340,"counts":{"ok":119888},"lifted":false,"calibration":{"acc":0.6365,"conf":0.6899,"ece":0.0638,"brier":0.4905,"over95":0.0123,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.032,"conf":0.0479,"acc":0.078},{"lo":0.1,"w":0.0261,"conf":0.1494,"acc":0.1772},{"lo":0.2,"w":0.0256,"conf":0.2554,"acc":0.2821},{"lo":0.3,"w":0.0595,"conf":0.3597,"acc":0.3631},{"lo":0.4,"w":0.0724,"conf":0.4517,"acc":0.4413},{"lo":0.5,"w":0.1393,"conf":0.5441,"acc":0.5627},{"lo":0.6,"w":0.0971,"conf":0.6497,"acc":0.5451},{"lo":0.7,"w":0.1044,"conf":0.7524,"acc":0.6601},{"lo":0.8,"w":0.1592,"conf":0.8537,"acc":0.735},{"lo":0.9,"w":0.2842,"conf":0.9513,"acc":0.8839}],"areas":{"knowledge":{"ece":0.1221,"acc":0.5537,"conf":0.6758},"language":{"ece":0.1019,"acc":0.7001,"conf":0.7358},"retrieval":{"ece":0.0647,"acc":0.672,"conf":0.6628},"tools":{"ece":0.047,"acc":0.8834,"conf":0.8426},"arts":{"ece":0.0521,"acc":0.5274,"conf":0.5684}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":70.8,"p95":687.7,"mean":399.5},"results":{"1":{"score":0.902,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":151.1},"2":{"score":0.6223,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":11519.9},"3":{"score":0.7913,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8675,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":103.9,"wrong_rows":22,"contamination":"22 BANKING77 test items found in lev's training records count as wrong"},"5":{"score":0.9037,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":176.0,"wrong_rows":12,"contamination":"12 CLINC150+OOS test items found in lev's training records count as wrong"},"6":{"score":0.7986,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":536.5},"9":{"score":0.1932,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.3398,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":122.4},"11":{"score":0.7043,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6445,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":69.5,"wrong_rows":1,"contamination":"1 ANLI test items found in lev's training records count as wrong"},"20":{"score":0.8766,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":69.4},"21":{"score":0.6008,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":68.8},"22":{"score":0.054,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":750.8},"23":{"score":0.6224,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":70.7},"24":{"score":0.7295,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":69.5},"25":{"score":0.4031,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9752,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":69.1,"wrong_rows":9,"contamination":"9 ARC-Easy test items found in lev's training records count as wrong"},"27":{"score":0.9326,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":69.1},"28":{"score":0.6685,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":68.8},"29":{"score":0.8118,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":70.4,"wrong_rows":1,"contamination":"1 HellaSwag test items found in lev's training records count as wrong"},"30":{"score":0.3245,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":69.7},"31":{"score":0.1432,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":101.0},"32":{"score":0.5359,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.1824,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":783.1},"36":{"score":0.4398,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4594,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":70.9},"38":{"score":0.0711,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1531.2},"39":{"score":0.7395,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":68.6,"tracks":[{"track":"A · Arabic","score":0.2566,"headline":false},{"track":"A · English","score":0.4,"headline":true},{"track":"C · Arabic pairs","score":0.6,"headline":false},{"track":"C · English pairs","score":0.885,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4836,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":69.8},"42":{"score":0.7606,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":71.3},"43":{"score":0.5035,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5966,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":69.5},"45":{"score":0.1238,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2158,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":82.2},"50":{"score":0.4624,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":94.9},"56":{"score":0.5055,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":194.7,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.4866,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":70.1,"edition_added":"v0.2","jev":0.827},"58":{"score":0.6134,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":69.7,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.5225,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":80.8,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.5398,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.0832,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":71.5,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.6338,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":71.1,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":53.16,"balanced_skill":38.54,"breadth_skill":36.82},"frozen_scores":{"balanced_raw":41.02,"balanced_skill":26.8,"breadth_skill":23.1},"categories":[{"id":"knowledge","raw":0.3924,"skill":0.2197,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.5464,"skill":0.3852,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.6442,"skill":0.4792,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.6241,"skill":0.5752,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4581,"skill":0.2796,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.902,"skill":0.8677,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6223,"skill":0.5638,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7986,"skill":0.5252,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7849,"headline":false},{"track":"RouterBench-5shot","score":0.8123,"headline":false}]},"11":{"raw":0.7043,"skill":0.5724,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.8753,"skill":0.7506,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6008,"skill":0.2017,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0524,"skill":0.045,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.6036,"skill":0.2071,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7444,"skill":0.6592,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4031,"skill":0.2041,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.3245,"skill":0.1867,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.4352,"headline":false},{"track":"GSM8K-10choice","score":0.2138,"headline":false}]},"31":{"raw":0.1432,"skill":0.0668,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4398,"skill":0.3663,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.4594,"skill":0.3219,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.4,"skill":0.2281,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2566,"headline":false},{"track":"A · English","score":0.4,"headline":true},{"track":"C · Arabic pairs","score":0.6,"headline":false},{"track":"C · English pairs","score":0.885,"headline":false}]},"41":{"raw":0.4836,"skill":0.2254,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.5035,"skill":0.2123,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5966,"skill":0.1932,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4624,"skill":0.2198,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5055,"skill":0.011,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.4866,"skill":0.4226,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6134,"skill":0.4396,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.5225,"skill":0.3633,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.5398,"skill":0.4247,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.0832,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.6338,"skill":0.2676,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5359,"skill":0.2622,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.1824,"skill":0.1715,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1238,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.6445,"skill":0.4675,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.6685,"skill":0.337,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.8118,"skill":0.7491,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0711,"skill":0.0414,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.7395,"skill":0.6169,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7606,"skill":0.5342,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.8675,"skill":0.8658,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.9037,"skill":0.9031,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.7913,"skill":0.7873,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.1932,"skill":0.1932,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.1368,"skill":0.1368,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author DecisionEngine.system_one single batched forward, bf16, LEV_LOGITS_KEEP=last (board setting), torch 2.14.0+cu130","contamination":[{"catalog_id":4,"benchmark":"BANKING77","wrong":22,"flag":"22 BANKING77 test items found in lev's training records count as wrong"},{"catalog_id":5,"benchmark":"CLINC150+OOS","wrong":12,"flag":"12 CLINC150+OOS test items found in lev's training records count as wrong"},{"catalog_id":12,"benchmark":"ANLI","wrong":1,"flag":"1 ANLI test items found in lev's training records count as wrong"},{"catalog_id":26,"benchmark":"ARC-Easy","wrong":9,"flag":"9 ARC-Easy test items found in lev's training records count as wrong"},{"catalog_id":29,"benchmark":"HellaSwag","wrong":1,"flag":"1 HellaSwag test items found in lev's training records count as wrong"}],"new_in_edition":true},{"engine":"intern-decision-0.8b","name":"Intern-Decision-0.8B","long_name":"InternLM Intern-Decision Qwen3.5-0.8B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":873438784,"params_estimated":false,"params_from_name":false,"trained_bytes":1706034736,"model_url":"https://huggingface.co/internlm/Intern-Decision-0.8B","code_url":"https://github.com/internlm/Intern-Decision","base_model":"Qwen/Qwen3.5-0.8B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B-Base","weights_repo":"internlm/Intern-Decision-0.8B","served_checkpoint":null},"completed":120340,"counts":{"ok":119888},"lifted":false,"calibration":{"acc":0.4516,"conf":0.4473,"ece":0.0255,"brier":0.6309,"over95":0,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0489,"conf":0.0817,"acc":0.0267},{"lo":0.1,"w":0.0995,"conf":0.1444,"acc":0.0845},{"lo":0.2,"w":0.0566,"conf":0.2589,"acc":0.2427},{"lo":0.3,"w":0.1324,"conf":0.3555,"acc":0.3671},{"lo":0.4,"w":0.1541,"conf":0.4474,"acc":0.4403},{"lo":0.5,"w":0.3364,"conf":0.5396,"acc":0.5602},{"lo":0.6,"w":0.1191,"conf":0.6397,"acc":0.6686},{"lo":0.7,"w":0.0387,"conf":0.7381,"acc":0.7769},{"lo":0.8,"w":0.0132,"conf":0.8369,"acc":0.9399},{"lo":0.9,"w":0.0012,"conf":0.9138,"acc":1.0}],"areas":{"knowledge":{"ece":0.0262,"acc":0.4155,"conf":0.4195},"language":{"ece":0.0463,"acc":0.543,"conf":0.5182},"retrieval":{"ece":0.0943,"acc":0.3738,"conf":0.3365},"tools":{"ece":0.0559,"acc":0.4619,"conf":0.4587},"arts":{"ece":0.0562,"acc":0.4224,"conf":0.463}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":33.7,"p95":39.2,"mean":36.6},"results":{"1":{"score":0.6446,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":35.4},"2":{"score":0.4742,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":83.9},"3":{"score":0.2106,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.1185,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":35.6},"5":{"score":0.049,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":38.0},"6":{"score":0.566,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":37.9},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5742,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":35.9},"11":{"score":0.3545,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.3409,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":33.2},"20":{"score":0.5036,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":33.2},"21":{"score":0.5369,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":32.8},"22":{"score":0.0155,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":71.0},"23":{"score":0.5674,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":34.2},"24":{"score":0.469,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":33.0},"25":{"score":0.2806,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.8123,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":33.0},"27":{"score":0.6288,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":32.9},"28":{"score":0.5312,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":32.7},"29":{"score":0.4356,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":33.2},"30":{"score":0.2843,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":33.3},"31":{"score":0.088,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":35.1},"32":{"score":0.5133,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0339,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":36.1},"36":{"score":0.2237,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.2436,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":33.5},"38":{"score":0.0464,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":82.8},"39":{"score":0.5708,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2628,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":32.8,"tracks":[{"track":"A · Arabic","score":0.2461,"headline":false},{"track":"A · English","score":0.2628,"headline":true},{"track":"C · Arabic pairs","score":0.545,"headline":false},{"track":"C · English pairs","score":0.615,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.2798,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":33.3},"42":{"score":0.5765,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":34.2},"43":{"score":0.3281,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.505,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":33.1},"45":{"score":0.1357,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2382,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":34.6},"50":{"score":0.4463,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":35.1},"56":{"score":0.4585,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":35.6,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.2654,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":33.3,"edition_added":"v0.2","jev":0.827},"58":{"score":0.4267,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":33.2,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.4608,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":35.5,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.2708,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.5326,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":34.8,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5235,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":34.1,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":32.68,"balanced_skill":11.94,"breadth_skill":10.89},"frozen_scores":{"balanced_raw":28.88,"balanced_skill":8.6,"breadth_skill":7.69},"categories":[{"id":"knowledge","raw":0.2854,"skill":0.0797,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.3955,"skill":0.1035,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.2568,"skill":0.0644,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.3632,"skill":0.2838,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.3297,"skill":0.0722,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.6446,"skill":0.5203,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.4742,"skill":0.3927,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.566,"skill":0.0174,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.5411,"headline":false},{"track":"RouterBench-5shot","score":0.5909,"headline":false}]},"11":{"raw":0.3545,"skill":0.0665,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.5026,"skill":0.0052,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5369,"skill":0.0738,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0148,"skill":0.0071,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5461,"skill":0.0923,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4827,"skill":0.3103,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2806,"skill":0.0408,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.2843,"skill":0.1369,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.3889,"headline":false},{"track":"GSM8K-10choice","score":0.1797,"headline":false}]},"31":{"raw":0.088,"skill":0.0067,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.2237,"skill":0.1218,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.2436,"skill":0.0513,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.2628,"skill":0.0516,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2461,"headline":false},{"track":"A · English","score":0.2628,"headline":true},{"track":"C · Arabic pairs","score":0.545,"headline":false},{"track":"C · English pairs","score":0.615,"headline":false}]},"41":{"raw":0.2798,"skill":0.0,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3281,"skill":0.0,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.505,"skill":0.01,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4463,"skill":0.1964,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.4585,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.2654,"skill":0.1738,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.4267,"skill":0.169,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.4608,"skill":0.2811,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.2708,"skill":0.0885,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.5326,"skill":0.0309,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5235,"skill":0.047,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5133,"skill":0.2262,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0339,"skill":0.0211,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1357,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.3409,"skill":0.0127,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5312,"skill":0.0624,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.4356,"skill":0.2475,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0464,"skill":0.0159,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.5708,"skill":0.3687,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.5765,"skill":0.1761,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.1185,"skill":0.1072,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.049,"skill":0.0433,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.2106,"skill":0.1954,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0472,"skill":0.0472,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author DecisionEngine.predict on the whole row, SDPA, max_length 262144 and the board lifts; transformers fast path enabled with causal-conv1d 1.7.0 (fla 0.4.2 gated delta + CUDA conv). The board ran the torch fallback because causal-conv1d was absent; that run is kept as latency-v1-full.torch-fallback. venv-intern torch 2.9.1+cu128","new_in_edition":true},{"engine":"intern-decision-2b","name":"Intern-Decision-2B","long_name":"InternLM Intern-Decision Qwen3.5-2B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":2274069824,"params_estimated":false,"params_from_name":false,"trained_bytes":4426562832,"model_url":"https://huggingface.co/internlm/Intern-Decision-2B","code_url":"https://github.com/internlm/Intern-Decision","base_model":"Qwen/Qwen3.5-2B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-2B-Base","weights_repo":"internlm/Intern-Decision-2B","served_checkpoint":null},"completed":120340,"counts":{"ok":119888},"lifted":false,"calibration":{"acc":0.4884,"conf":0.5492,"ece":0.0608,"brier":0.6076,"over95":0.0002,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0269,"conf":0.0845,"acc":0.027},{"lo":0.1,"w":0.0567,"conf":0.145,"acc":0.0571},{"lo":0.2,"w":0.0583,"conf":0.246,"acc":0.1486},{"lo":0.3,"w":0.0778,"conf":0.3538,"acc":0.2459},{"lo":0.4,"w":0.115,"conf":0.4514,"acc":0.4273},{"lo":0.5,"w":0.2544,"conf":0.5451,"acc":0.5007},{"lo":0.6,"w":0.175,"conf":0.6452,"acc":0.5615},{"lo":0.7,"w":0.1054,"conf":0.7486,"acc":0.6834},{"lo":0.8,"w":0.0864,"conf":0.8488,"acc":0.8061},{"lo":0.9,"w":0.0441,"conf":0.933,"acc":0.9125}],"areas":{"knowledge":{"ece":0.0634,"acc":0.4866,"conf":0.5355},"language":{"ece":0.0864,"acc":0.5476,"conf":0.6279},"retrieval":{"ece":0.1024,"acc":0.3515,"conf":0.4539},"tools":{"ece":0.0813,"acc":0.5643,"conf":0.577},"arts":{"ece":0.052,"acc":0.4502,"conf":0.4975}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":33.5,"p95":58.0,"mean":39.2},"results":{"1":{"score":0.7102,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":35.3},"2":{"score":0.4904,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":128.5},"3":{"score":0.2854,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.1291,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":35.6},"5":{"score":0.0426,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":38.3},"6":{"score":0.7934,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":46.3},"9":{"score":0.0114,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5929,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":35.9},"11":{"score":0.4477,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.4694,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":32.7},"20":{"score":0.589,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":32.7},"21":{"score":0.578,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":32.5},"22":{"score":0.0085,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":110.0},"23":{"score":0.5582,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":33.5},"24":{"score":0.5727,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":32.5},"25":{"score":0.3418,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9057,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":32.5},"27":{"score":0.7816,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":32.5},"28":{"score":0.5754,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":32.4},"29":{"score":0.6043,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":33.1},"30":{"score":0.4098,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":33.0},"31":{"score":0.106,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":34.8},"32":{"score":0.5439,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.1121,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":35.9},"36":{"score":0.2137,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3463,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":33.5},"38":{"score":0.0402,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":123.2},"39":{"score":0.6317,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2867,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":32.5,"tracks":[{"track":"A · Arabic","score":0.2334,"headline":false},{"track":"A · English","score":0.2867,"headline":true},{"track":"C · Arabic pairs","score":0.515,"headline":false},{"track":"C · English pairs","score":0.585,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3933,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":32.7},"42":{"score":0.6628,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":33.8},"43":{"score":0.3386,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5378,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":32.6},"45":{"score":0.1277,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2387,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":34.1},"50":{"score":0.457,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":34.6},"56":{"score":0.7,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":35.2,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.33,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":33.1,"edition_added":"v0.2","jev":0.827},"58":{"score":0.4732,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":33.0,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.4907,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":35.1,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.4034,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.2969,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":34.4,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5675,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":34.1,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":37.45,"balanced_skill":19.38,"breadth_skill":18.65},"frozen_scores":{"balanced_raw":33.16,"balanced_skill":15.05,"breadth_skill":13.03},"categories":[{"id":"knowledge","raw":0.3311,"skill":0.138,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.447,"skill":0.2056,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.3146,"skill":0.1566,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.405,"skill":0.3334,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.3639,"skill":0.1272,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.7102,"skill":0.6087,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.4904,"skill":0.4114,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7934,"skill":0.5119,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7836,"headline":false},{"track":"RouterBench-5shot","score":0.8032,"headline":false}]},"11":{"raw":0.4477,"skill":0.2013,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.5851,"skill":0.1701,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.578,"skill":0.156,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0085,"skill":0.0008,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5341,"skill":0.0682,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5921,"skill":0.4561,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3418,"skill":0.1224,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.4098,"skill":0.287,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.4852,"headline":false},{"track":"GSM8K-10choice","score":0.3343,"headline":false}]},"31":{"raw":0.106,"skill":0.0263,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.2137,"skill":0.1104,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.3463,"skill":0.1801,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.2867,"skill":0.0824,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2334,"headline":false},{"track":"A · English","score":0.2867,"headline":true},{"track":"C · Arabic pairs","score":0.515,"headline":false},{"track":"C · English pairs","score":0.585,"headline":false}]},"41":{"raw":0.3933,"skill":0.0899,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3386,"skill":0.0,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5378,"skill":0.0756,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.457,"skill":0.212,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.7,"skill":0.4,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.33,"skill":0.2464,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.4732,"skill":0.2364,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.4907,"skill":0.3209,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.4034,"skill":0.2542,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.2969,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5675,"skill":0.135,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5439,"skill":0.2749,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.1121,"skill":0.1003,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1277,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.4694,"skill":0.2052,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5754,"skill":0.1508,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.6043,"skill":0.4724,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0402,"skill":0.0095,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.6317,"skill":0.4583,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.6628,"skill":0.344,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.1291,"skill":0.1179,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.0426,"skill":0.0368,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.2854,"skill":0.2716,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0114,"skill":0.0114,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0452,"skill":0.0452,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author DecisionEngine.predict on the whole row, SDPA, max_length 262144 and the board lifts; transformers fast path enabled with causal-conv1d 1.7.0 (fla 0.4.2 gated delta + CUDA conv). The board ran the torch fallback because causal-conv1d was absent; that run is kept as latency-v1-full.torch-fallback. venv-intern torch 2.9.1+cu128","new_in_edition":true},{"engine":"intern-decision-4b","name":"Intern-Decision-4B","long_name":"InternLM Intern-Decision Qwen3.5-4B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":9078626736,"model_url":"https://huggingface.co/internlm/Intern-Decision-4B","code_url":"https://github.com/internlm/Intern-Decision","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"internlm/Intern-Decision-4B","served_checkpoint":null},"completed":120340,"counts":{"ok":119888},"lifted":false,"calibration":{"acc":0.6355,"conf":0.6633,"ece":0.0278,"brier":0.4737,"over95":0.0025,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0193,"conf":0.0783,"acc":0.0288},{"lo":0.1,"w":0.0374,"conf":0.1445,"acc":0.1325},{"lo":0.2,"w":0.0286,"conf":0.2507,"acc":0.2472},{"lo":0.3,"w":0.048,"conf":0.3511,"acc":0.3018},{"lo":0.4,"w":0.0715,"conf":0.4555,"acc":0.4349},{"lo":0.5,"w":0.1605,"conf":0.5478,"acc":0.5165},{"lo":0.6,"w":0.1491,"conf":0.6497,"acc":0.631},{"lo":0.7,"w":0.1475,"conf":0.751,"acc":0.703},{"lo":0.8,"w":0.1634,"conf":0.8534,"acc":0.8175},{"lo":0.9,"w":0.1746,"conf":0.9434,"acc":0.9339}],"areas":{"knowledge":{"ece":0.064,"acc":0.5564,"conf":0.6199},"language":{"ece":0.034,"acc":0.7187,"conf":0.7509},"retrieval":{"ece":0.0692,"acc":0.6735,"conf":0.6306},"tools":{"ece":0.092,"acc":0.8465,"conf":0.7553},"arts":{"ece":0.0689,"acc":0.5029,"conf":0.5718}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":44.2,"p95":115.4,"mean":60.0},"results":{"1":{"score":0.9274,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":46.2},"2":{"score":0.5282,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":286.4},"3":{"score":0.7598,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.6261,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":47.2},"5":{"score":0.7273,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":65.8},"6":{"score":0.7984,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":98.9},"9":{"score":0.125,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.665,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":63.4},"11":{"score":0.6202,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5839,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":43.4},"20":{"score":0.8182,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":43.3},"21":{"score":0.5864,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":43.2},"22":{"score":0.07,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":239.1},"23":{"score":0.5854,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":44.4},"24":{"score":0.7137,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":43.2},"25":{"score":0.4133,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9663,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":43.0},"27":{"score":0.9138,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":43.2},"28":{"score":0.6835,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":43.5},"29":{"score":0.7755,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":43.5},"30":{"score":0.4621,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":43.8},"31":{"score":0.1262,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":46.0},"32":{"score":0.5372,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3079,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":48.0},"36":{"score":0.331,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.453,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":44.0},"38":{"score":0.1587,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":274.3},"39":{"score":0.8087,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4636,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":43.4,"tracks":[{"track":"A · Arabic","score":0.2965,"headline":false},{"track":"A · English","score":0.4636,"headline":true},{"track":"C · Arabic pairs","score":0.675,"headline":false},{"track":"C · English pairs","score":0.895,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4066,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":43.3},"42":{"score":0.7783,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":44.8},"43":{"score":0.4509,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5998,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":43.3},"45":{"score":0.1277,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2111,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":45.2},"50":{"score":0.4511,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":45.6},"56":{"score":0.615,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":46.2,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.4614,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":43.5,"edition_added":"v0.2","jev":0.827},"58":{"score":0.6207,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":43.3,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.5775,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":46.8,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.5852,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.7186,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":45.3,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.6052,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":44.7,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":52.99,"balanced_skill":37.81,"breadth_skill":36.48},"frozen_scores":{"balanced_raw":39.66,"balanced_skill":24.67,"breadth_skill":21.27},"categories":[{"id":"knowledge","raw":0.4104,"skill":0.2371,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.6044,"skill":0.4188,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.5598,"skill":0.4034,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.6028,"skill":0.5566,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4534,"skill":0.2608,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9274,"skill":0.902,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.5282,"skill":0.4551,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7984,"skill":0.5245,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7857,"headline":false},{"track":"RouterBench-5shot","score":0.8111,"headline":false}]},"11":{"raw":0.6202,"skill":0.4508,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.8202,"skill":0.6405,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5864,"skill":0.1728,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.069,"skill":0.0617,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5656,"skill":0.1311,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7235,"skill":0.6314,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4133,"skill":0.2177,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.4621,"skill":0.347,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.5019,"headline":false},{"track":"GSM8K-10choice","score":0.4223,"headline":false}]},"31":{"raw":0.1262,"skill":0.0483,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.331,"skill":0.2432,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.453,"skill":0.3139,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.4636,"skill":0.3099,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2965,"headline":false},{"track":"A · English","score":0.4636,"headline":true},{"track":"C · Arabic pairs","score":0.675,"headline":false},{"track":"C · English pairs","score":0.895,"headline":false}]},"41":{"raw":0.4066,"skill":0.11,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4509,"skill":0.1288,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5998,"skill":0.1996,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4511,"skill":0.2033,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.615,"skill":0.23,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.4614,"skill":0.3942,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6207,"skill":0.4502,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.5775,"skill":0.4367,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.5852,"skill":0.4815,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.7186,"skill":0.4165,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.6052,"skill":0.2104,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5372,"skill":0.2642,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.3079,"skill":0.2987,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1277,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.5839,"skill":0.3767,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.6835,"skill":0.367,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.7755,"skill":0.7007,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.1587,"skill":0.1318,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8087,"skill":0.7186,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7783,"skill":0.5687,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.6261,"skill":0.6213,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.7273,"skill":0.7257,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.7598,"skill":0.7552,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.125,"skill":0.125,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.1556,"skill":0.1556,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author DecisionEngine.predict on the whole row, SDPA, max_length 262144 and the board lifts; transformers fast path enabled with causal-conv1d 1.7.0 (fla 0.4.2 gated delta + CUDA conv). The board ran the torch fallback because causal-conv1d was absent; that run is kept as latency-v1-full.torch-fallback. venv-intern torch 2.9.1+cu128","new_in_edition":true},{"engine":"simple-jev-qwen3.8-27b","name":"simple-jev · Qwen3.8-27B (featherless)","long_name":"simple-jev · Qwen3.8-27B (featherless)","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/featherless-ai/simple-jev","base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":"Qwen/Qwen3.8-27B"},"completed":120340,"counts":{"ok":119888},"lifted":false,"calibration":{"acc":0.72,"conf":0.8322,"ece":0.113,"brier":0.4001,"over95":0.0385,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0045,"conf":0.0736,"acc":0.0638},{"lo":0.1,"w":0.0112,"conf":0.1491,"acc":0.1851},{"lo":0.2,"w":0.0127,"conf":0.2513,"acc":0.2158},{"lo":0.3,"w":0.0275,"conf":0.3543,"acc":0.3127},{"lo":0.4,"w":0.0461,"conf":0.4542,"acc":0.3266},{"lo":0.5,"w":0.0789,"conf":0.5525,"acc":0.446},{"lo":0.6,"w":0.0686,"conf":0.65,"acc":0.4865},{"lo":0.7,"w":0.0789,"conf":0.7502,"acc":0.5498},{"lo":0.8,"w":0.0913,"conf":0.8558,"acc":0.6403},{"lo":0.9,"w":0.5804,"conf":0.9836,"acc":0.8975}],"areas":{"knowledge":{"ece":0.1302,"acc":0.6437,"conf":0.7739},"language":{"ece":0.066,"acc":0.8335,"conf":0.8995},"retrieval":{"ece":0.1885,"acc":0.6781,"conf":0.8652},"tools":{"ece":0.0431,"acc":0.9373,"conf":0.9803},"arts":{"ece":0.1477,"acc":0.5866,"conf":0.7241}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":373.0,"p95":2796.8,"mean":930.6},"results":{"1":{"score":0.9681,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":494.0},"2":{"score":0.6798,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":21436.8},"3":{"score":0.8543,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7946,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":1165.1},"5":{"score":0.8808,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":2429.1},"6":{"score":0.7978,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":992.3},"9":{"score":0.6477,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4189,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":608.5},"11":{"score":0.7753,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6923,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":347.3},"20":{"score":0.9612,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":320.7},"21":{"score":0.613,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":282.4},"22":{"score":0.2895,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":4623.1},"23":{"score":0.6362,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":409.8},"24":{"score":0.8401,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":313.4},"25":{"score":0.4847,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9924,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":308.2},"27":{"score":0.9684,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":310.1},"28":{"score":0.8445,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":280.1},"29":{"score":0.931,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":367.5},"30":{"score":0.6524,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":328.0},"31":{"score":0.1736,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":698.6},"32":{"score":0.6316,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3206,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":637.4},"36":{"score":0.4974,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5735,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":351.0},"38":{"score":0.3558,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":6283.9},"39":{"score":0.8848,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.6035,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":282.2,"tracks":[{"track":"A · Arabic","score":0.3493,"headline":false},{"track":"A · English","score":0.6035,"headline":true},{"track":"C · Arabic pairs","score":0.805,"headline":false},{"track":"C · English pairs","score":0.97,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.7185,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":349.6},"42":{"score":0.8374,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":392.5},"43":{"score":0.6228,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.7102,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":334.6},"45":{"score":0.1238,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.213,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":419.4},"50":{"score":0.4105,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":537.1},"56":{"score":0.8465,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":3020.8,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.5975,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":360.5,"edition_added":"v0.2","jev":0.827},"58":{"score":0.708,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":311.7,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.7503,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":485.6,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.6875,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.7233,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":392.9,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.7462,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":359.2,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":66.26,"balanced_skill":55.74,"breadth_skill":53.91},"frozen_scores":{"balanced_raw":48.32,"balanced_skill":37.01,"breadth_skill":31.06},"categories":[{"id":"knowledge","raw":0.5007,"skill":0.3656,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.7415,"skill":0.6209,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.7233,"skill":0.6325,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.7898,"skill":0.7621,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.5232,"skill":0.3648,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9681,"skill":0.957,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6798,"skill":0.6303,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7979,"skill":0.5234,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7843,"headline":false},{"track":"RouterBench-5shot","score":0.8114,"headline":false}]},"11":{"raw":0.7753,"skill":0.675,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.9627,"skill":0.9253,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.613,"skill":0.226,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.2826,"skill":0.277,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.6334,"skill":0.2668,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8425,"skill":0.7899,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4847,"skill":0.3129,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.6524,"skill":0.5768,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.6672,"headline":false},{"track":"GSM8K-10choice","score":0.6376,"headline":false}]},"31":{"raw":0.1736,"skill":0.0999,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4974,"skill":0.4314,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.5735,"skill":0.4651,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.6035,"skill":0.4899,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3493,"headline":false},{"track":"A · English","score":0.6035,"headline":true},{"track":"C · Arabic pairs","score":0.805,"headline":false},{"track":"C · English pairs","score":0.97,"headline":false}]},"41":{"raw":0.7185,"skill":0.5778,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.6228,"skill":0.4015,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.7102,"skill":0.4204,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4105,"skill":0.1444,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.8465,"skill":0.693,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.5975,"skill":0.5473,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.708,"skill":0.5768,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.7503,"skill":0.6671,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.6875,"skill":0.6094,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.7233,"skill":0.4263,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.7462,"skill":0.4924,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.6316,"skill":0.4143,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.3206,"skill":0.3116,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1238,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.6923,"skill":0.5391,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.8445,"skill":0.689,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.931,"skill":0.908,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.3558,"skill":0.3352,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8848,"skill":0.8306,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.8374,"skill":0.6837,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.7946,"skill":0.792,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.8808,"skill":0.8801,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.8543,"skill":0.8515,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.6477,"skill":0.6477,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.148,"skill":0.148,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author hf-server/hf_server.py (bf16, fla kernels, examples_binary policy, max-model-len 65536, batch tokens 65536), one sequential client (JEV_ATTACH); 203 of 760 rows ran out of GPU memory at the server default batch and were answered in-process by the same hf_server service with max_batch_size 32 (simplejev_ladder.py, as on the board for heavy rows); causal-conv1d absent (torch conv fallback, as on the board)","new_in_edition":true},{"engine":"neohorse-jev-4b","name":"NeoHorse-Jev-4B","long_name":"TokenRhythm NeoHorse-Jev-4B","variant":null,"meta":{"kind":"head / adapter","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":9083860816,"model_url":"https://huggingface.co/TokenRhythm/NeoHorse-Jev-4B","code_url":"https://github.com/TokenRhythm/NeoHorse/tree/main/jev","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"TokenRhythm/NeoHorse-Jev-4B","served_checkpoint":null},"completed":120340,"counts":{"ok":119888},"lifted":false,"calibration":{"acc":0.6071,"conf":0.711,"ece":0.1039,"brier":0.5134,"over95":0.0277,"n":35088,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0413,"conf":0.0732,"acc":0.0537},{"lo":0.1,"w":0.0279,"conf":0.1459,"acc":0.1244},{"lo":0.2,"w":0.0236,"conf":0.2574,"acc":0.174},{"lo":0.3,"w":0.051,"conf":0.3549,"acc":0.3136},{"lo":0.4,"w":0.0671,"conf":0.454,"acc":0.3694},{"lo":0.5,"w":0.103,"conf":0.5488,"acc":0.4829},{"lo":0.6,"w":0.1012,"conf":0.652,"acc":0.5617},{"lo":0.7,"w":0.1072,"conf":0.7506,"acc":0.634},{"lo":0.8,"w":0.1342,"conf":0.8512,"acc":0.6751},{"lo":0.9,"w":0.3435,"conf":0.9667,"acc":0.8483}],"areas":{"knowledge":{"ece":0.0809,"acc":0.5452,"conf":0.6255},"language":{"ece":0.1292,"acc":0.6703,"conf":0.7995},"retrieval":{"ece":0.0726,"acc":0.7064,"conf":0.7543},"tools":{"ece":0.0689,"acc":0.8587,"conf":0.8892},"arts":{"ece":0.1992,"acc":0.3991,"conf":0.5982}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":49.0,"p95":273.9,"mean":208.6},"results":{"1":{"score":0.9622,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":64.5},"2":{"score":0.6453,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":8815.4},"3":{"score":0.6614,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8379,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":52.6},"5":{"score":0.7812,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":71.6},"6":{"score":0.7947,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":191.7},"9":{"score":0.3068,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5987,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":56.7},"11":{"score":0.7256,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5189,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":47.1},"20":{"score":0.5054,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":47.2},"21":{"score":0.5175,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":46.4},"22":{"score":0.063,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":302.3},"23":{"score":0.5718,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":49.2},"24":{"score":0.6786,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":47.5},"25":{"score":0.3214,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.963,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":46.9},"27":{"score":0.8916,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":47.1},"28":{"score":0.6867,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":46.2},"29":{"score":0.722,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":47.7},"30":{"score":0.3825,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":47.5},"31":{"score":0.11,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":51.1},"32":{"score":0.5612,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2158,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":467.5},"36":{"score":0.4079,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4435,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":48.7},"38":{"score":0.0536,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":383.6},"39":{"score":0.8192,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.3804,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":46.3,"tracks":[{"track":"A · Arabic","score":0.3177,"headline":false},{"track":"A · English","score":0.3804,"headline":true},{"track":"C · Arabic pairs","score":0.39,"headline":false},{"track":"C · English pairs","score":0.685,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4739,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":47.4},"42":{"score":0.7498,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":49.8},"43":{"score":0.4965,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6494,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":47.3},"45":{"score":0.1078,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2113,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":50.4},"50":{"score":0.3383,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":51.8},"56":{"score":0.5375,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":84.0,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.4541,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":47.6,"edition_added":"v0.2","jev":0.827},"58":{"score":0.6368,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":47.3,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.5183,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":50.9,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.5265,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.6848,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":50.9,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.593,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":49.2,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":52.73,"balanced_skill":36.75,"breadth_skill":34.26},"frozen_scores":{"balanced_raw":38.1,"balanced_skill":21.59,"breadth_skill":16.88},"categories":[{"id":"knowledge","raw":0.3925,"skill":0.2259,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.5849,"skill":0.3936,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.607,"skill":0.4391,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.6331,"skill":0.5888,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.3737,"skill":0.1177,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9622,"skill":0.949,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6453,"skill":0.5903,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7947,"skill":0.5154,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.783,"headline":false},{"track":"RouterBench-5shot","score":0.8065,"headline":false}]},"11":{"raw":0.7256,"skill":0.6031,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.5083,"skill":0.0166,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5175,"skill":0.035,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0586,"skill":0.0512,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5554,"skill":0.1109,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6918,"skill":0.589,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3214,"skill":0.0952,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.3825,"skill":0.2557,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.4769,"headline":false},{"track":"GSM8K-10choice","score":0.2881,"headline":false}]},"31":{"raw":0.11,"skill":0.0307,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4079,"skill":0.3302,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.4435,"skill":0.3019,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.3804,"skill":0.2029,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3177,"headline":false},{"track":"A · English","score":0.3804,"headline":true},{"track":"C · Arabic pairs","score":0.39,"headline":false},{"track":"C · English pairs","score":0.685,"headline":false}]},"41":{"raw":0.4739,"skill":0.2109,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4965,"skill":0.2011,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6494,"skill":0.2988,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3383,"skill":0.0397,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5375,"skill":0.075,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.4541,"skill":0.386,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6368,"skill":0.4735,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.5183,"skill":0.3577,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.5265,"skill":0.4081,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.6848,"skill":0.3465,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.593,"skill":0.186,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5612,"skill":0.3024,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2158,"skill":0.2054,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1078,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.5189,"skill":0.2794,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.6867,"skill":0.3734,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.722,"skill":0.6293,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0536,"skill":0.0233,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8192,"skill":0.7341,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7498,"skill":0.5132,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.8379,"skill":0.8358,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.7812,"skill":0.7799,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.6614,"skill":0.6549,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.3068,"skill":0.3068,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.1548,"skill":0.1548,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author neohorse_decision 1.0.0 wheel in its own env (py3.12.10, torch 2.8.0+cu128, triton 3.7.1), board guard lifts, one process","new_in_edition":true},{"engine":"julia-1","name":"Julia 1","long_name":"Supersonic Labs Julia 1","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":140908331,"params_estimated":true,"params_from_name":false,"trained_bytes":577189056,"model_url":"https://huggingface.co/SupersonicLabs/Julia-1","code_url":null,"base_model":"jhu-clsp/mmBERT-small","base_url":"https://huggingface.co/jhu-clsp/mmBERT-small","weights_repo":"SupersonicLabs/Julia-1","served_checkpoint":null},"completed":120340,"counts":{"ok":119878,"unsupported":10},"lifted":false,"calibration":{"acc":0.3741,"conf":0.7938,"ece":0.4197,"brier":0.9866,"over95":0.1866,"n":35087,"benchmarks":32,"rel":[{"lo":0.0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0047,"conf":0.1592,"acc":0.1205},{"lo":0.2,"w":0.0139,"conf":0.2533,"acc":0.1083},{"lo":0.3,"w":0.0319,"conf":0.3527,"acc":0.1378},{"lo":0.4,"w":0.0522,"conf":0.4533,"acc":0.2241},{"lo":0.5,"w":0.0987,"conf":0.549,"acc":0.3663},{"lo":0.6,"w":0.1025,"conf":0.6506,"acc":0.3412},{"lo":0.7,"w":0.1133,"conf":0.7511,"acc":0.3826},{"lo":0.8,"w":0.1439,"conf":0.8525,"acc":0.386},{"lo":0.9,"w":0.4389,"conf":0.9705,"acc":0.4236}],"areas":{"knowledge":{"ece":0.434,"acc":0.3558,"conf":0.7898},"language":{"ece":0.3759,"acc":0.4373,"conf":0.8132},"retrieval":{"ece":0.3899,"acc":0.3611,"conf":0.751},"tools":{"ece":0.5319,"acc":0.2752,"conf":0.8038},"arts":{"ece":0.4256,"acc":0.3703,"conf":0.7958}}},"gaps":{"share":0.0001,"reasons":[{"reason":"prompt longer than its 8,192-token context window (no truncation allowed)","share":0.0001}]},"latency":{"median":5.8,"p95":15.3,"mean":72.7},"results":{"1":{"score":0.3749,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":6.4},"2":{"score":0.1358,"jev":0.6535,"metric":"nDCG@10","requests":685,"answered":680,"unsupported":5,"errors":0,"median_ms":null},"3":{"score":0.1831,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.426,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":5.7},"5":{"score":0.1234,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":6.0},"6":{"score":0.5756,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":12.5},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.3505,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":6.9},"11":{"score":0.3178,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.2864,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":5.5},"20":{"score":0.4878,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":5.4},"21":{"score":0.4825,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":5.3},"22":{"score":0.002,"jev":0.1813,"metric":"accuracy","requests":2000,"answered":1997,"unsupported":3,"errors":0,"median_ms":18.9},"23":{"score":0.5042,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":5.7},"24":{"score":0.3015,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":5.5},"25":{"score":0.2551,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.391,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":5.4},"27":{"score":0.3259,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":5.5},"28":{"score":0.5067,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":5.4},"29":{"score":0.2988,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":5.8},"30":{"score":0.6444,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":5.6},"31":{"score":0.0752,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":6.4},"32":{"score":0.3896,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0097,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":10.9},"36":{"score":0.0914,"jev":0.4765,"metric":"nDCG@10","requests":220,"answered":219,"unsupported":1,"errors":0,"median_ms":null},"37":{"score":0.206,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":5.6},"38":{"score":0.0312,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":28.6},"39":{"score":0.3279,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.0653,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":5.3,"tracks":[{"track":"A · Arabic","score":0.2039,"headline":false},{"track":"A · English","score":0.0653,"headline":true},{"track":"C · Arabic pairs","score":0.465,"headline":false},{"track":"C · English pairs","score":0.43,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3293,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":5.5},"42":{"score":0.4973,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":5.9},"43":{"score":0.3807,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5018,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":5.5},"45":{"score":0.144,"jev":0.204,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":null},"48":{"score":0.411,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":6.2},"50":{"score":0.3538,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":6.9},"56":{"score":0.4985,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":5.9,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.1467,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":5.8,"edition_added":"v0.2","jev":0.827},"58":{"score":0.3379,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":5.6,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.236,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":6.6,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.2348,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.5204,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":6.4,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.503,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":5.9,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":28.01,"balanced_skill":5.54,"breadth_skill":5.21},"frozen_scores":{"balanced_raw":25.29,"balanced_skill":3.82,"breadth_skill":3.15},"categories":[{"id":"knowledge","raw":0.2835,"skill":0.0661,"coverage":0.9998,"pending":0,"n":10},{"id":"language","raw":0.3207,"skill":0.0139,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.2994,"skill":0.0991,"coverage":0.9992,"pending":0,"n":6},{"id":"tools","raw":0.1927,"skill":0.072,"coverage":0.9986,"pending":0,"n":5},{"id":"arts","raw":0.2875,"skill":0.0174,"coverage":0.9998,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.3749,"skill":0.1561,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.1349,"skill":0.0009,"coverage":0.9927,"random":0.1341,"tracks":[]},"6":{"raw":0.5757,"skill":0.0328,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.504,"headline":false},{"track":"RouterBench-5shot","score":0.6473,"headline":false}]},"11":{"raw":0.3178,"skill":0.0134,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.487,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.4825,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0023,"skill":0.0,"coverage":0.9985,"random":0.0078,"tracks":[]},"23":{"raw":0.5097,"skill":0.0194,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3092,"skill":0.0789,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2551,"skill":0.0068,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.6444,"skill":0.5587,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.5838,"headline":false},{"track":"GSM8K-10choice","score":0.7051,"headline":false}]},"31":{"raw":0.0752,"skill":0.0,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.091,"skill":0.0,"coverage":0.9955,"random":0.116,"tracks":[]},"37":{"raw":0.206,"skill":0.0041,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.0653,"skill":0.0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2039,"headline":false},{"track":"A · English","score":0.0653,"headline":true},{"track":"C · Arabic pairs","score":0.465,"headline":false},{"track":"C · English pairs","score":0.43,"headline":false}]},"41":{"raw":0.3293,"skill":0.0,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3807,"skill":0.0174,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5018,"skill":0.0036,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3538,"skill":0.0622,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.4985,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.1467,"skill":0.0403,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.3379,"skill":0.0403,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.236,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.2348,"skill":0.0435,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.5204,"skill":0.0056,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.503,"skill":0.006,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.3896,"skill":0.0296,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0097,"skill":0.0,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1437,"skill":0.0,"coverage":0.998,"random":0.1641,"tracks":[]},"12":{"raw":0.2864,"skill":0.0,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5067,"skill":0.0134,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.2988,"skill":0.0651,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0312,"skill":0.0002,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.3279,"skill":0.0115,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.4973,"skill":0.022,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.426,"skill":0.4186,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.1234,"skill":0.1181,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.1831,"skill":0.1674,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.7595,"pending":0.24,"latency_rows":749,"latency_path":"author julia runtime (supersonic-julia 0.1.0, FastEngine) on CUDA with its optional compile_model=True (torch.compile dynamic=True; recompiles on new shapes are inside the timed rows), BF16 autocast, board guard lifts, one process; torch 2.14.0+cu130, transformers 5.0.0. Other variants kept as latency-v1-full.<variant>: eager (median 12.4 / p95 35.0 ms), marker_only_head (12.3 / 39.5), compile+marker_only_head (5.9 / 14.6)","new_in_edition":true},{"engine":"jebadiah-27b","name":"Jebadiah 27B","long_name":"frontier-infra Jebadiah 27B","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/frontier-infra/jebadiah-27b","code_url":"https://github.com/getainode/jebadiah","base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":"frontier-infra/jebadiah-27b","served_checkpoint":"1c0d794fc24b24b15175c6a912654fa5cd76e1bb"},"completed":120340,"counts":{"ok":120340},"lifted":false,"calibration":{"acc":0.7198,"conf":0.7281,"ece":0.014,"brier":0.3653,"over95":0.0065,"n":35093,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0171,"conf":0.0755,"acc":0.0734},{"lo":0.1,"w":0.0317,"conf":0.1452,"acc":0.1556},{"lo":0.2,"w":0.0272,"conf":0.2541,"acc":0.2635},{"lo":0.3,"w":0.0472,"conf":0.3499,"acc":0.3063},{"lo":0.4,"w":0.0542,"conf":0.4522,"acc":0.4446},{"lo":0.5,"w":0.1048,"conf":0.5474,"acc":0.4994},{"lo":0.6,"w":0.0954,"conf":0.6503,"acc":0.6122},{"lo":0.7,"w":0.1192,"conf":0.7518,"acc":0.757},{"lo":0.8,"w":0.1555,"conf":0.8537,"acc":0.8579},{"lo":0.9,"w":0.3478,"conf":0.9561,"acc":0.9588}],"areas":{"knowledge":{"ece":0.0344,"acc":0.6464,"conf":0.657},"language":{"ece":0.0114,"acc":0.8256,"conf":0.8287},"retrieval":{"ece":0.041,"acc":0.7143,"conf":0.7287},"tools":{"ece":0.0261,"acc":0.9399,"conf":0.9334},"arts":{"ece":0.0422,"acc":0.5629,"conf":0.58}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":110.4,"p95":591.1,"mean":220.5},"results":{"1":{"score":0.9758,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":239.3},"2":{"score":0.6606,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1974.5},"3":{"score":0.8406,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7871,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":194.8},"5":{"score":0.8828,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":310.0},"6":{"score":0.7988,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":508.4},"9":{"score":0.7273,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.589,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":236.5},"11":{"score":0.7762,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7148,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":97.3},"20":{"score":0.936,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":96.9},"21":{"score":0.6176,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":97.4},"22":{"score":0.2245,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":847.1},"23":{"score":0.6572,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":118.5},"24":{"score":0.8406,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":97.2},"25":{"score":0.4796,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9916,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":98.9},"27":{"score":0.9701,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":99.4},"28":{"score":0.8437,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":99.4},"29":{"score":0.9357,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":101.6},"30":{"score":0.6092,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":99.3},"31":{"score":0.1496,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":163.0},"32":{"score":0.6197,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.303,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":301.4},"36":{"score":0.4951,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5438,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":109.8},"38":{"score":0.2076,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1532.2},"39":{"score":0.8923,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5681,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":99.5,"tracks":[{"track":"A · Arabic","score":0.4813,"headline":false},{"track":"A · English","score":0.5681,"headline":true},{"track":"C · Arabic pairs","score":0.78,"headline":false},{"track":"C · English pairs","score":0.955,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.6385,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":99.8},"42":{"score":0.8316,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":137.6},"43":{"score":0.7281,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.7272,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":96.6},"45":{"score":0.0918,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.1782,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":169.6},"50":{"score":0.4248,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":190.9},"56":{"score":0.607,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":402.0,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.6271,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":98.4,"edition_added":"v0.2","jev":0.827},"58":{"score":0.777,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":98.1,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.7883,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":153.8,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.7235,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.7822,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":177.5,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.7003,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":118.8,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":65.62,"balanced_skill":54.67,"breadth_skill":53.18},"frozen_scores":{"balanced_raw":47.89,"balanced_skill":36.7,"breadth_skill":30.54},"categories":[{"id":"knowledge","raw":0.5099,"skill":0.3884,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.7254,"skill":0.6068,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.6741,"skill":0.5389,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.8066,"skill":0.7813,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.544,"skill":0.3873,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9758,"skill":0.9673,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6606,"skill":0.6081,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7988,"skill":0.5255,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7859,"headline":false},{"track":"RouterBench-5shot","score":0.8117,"headline":false}]},"11":{"raw":0.7762,"skill":0.6764,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.939,"skill":0.8779,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.6176,"skill":0.2352,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.2166,"skill":0.2104,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.6505,"skill":0.3009,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8469,"skill":0.7958,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4796,"skill":0.3061,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.6092,"skill":0.5326,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.7013,"headline":false},{"track":"GSM8K-10choice","score":0.5171,"headline":false}]},"31":{"raw":0.1496,"skill":0.0738,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4951,"skill":0.4288,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.5438,"skill":0.4277,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.5681,"skill":0.4444,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4813,"headline":false},{"track":"A · English","score":0.5681,"headline":true},{"track":"C · Arabic pairs","score":0.78,"headline":false},{"track":"C · English pairs","score":0.955,"headline":false}]},"41":{"raw":0.6385,"skill":0.4577,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.7281,"skill":0.5686,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.7272,"skill":0.4544,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4248,"skill":0.1652,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.607,"skill":0.214,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.6271,"skill":0.5806,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.777,"skill":0.6768,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.7883,"skill":0.7177,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.7235,"skill":0.6544,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.7822,"skill":0.5484,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.7003,"skill":0.4006,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.6197,"skill":0.3954,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.303,"skill":0.2937,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.0918,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.7148,"skill":0.5728,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.8437,"skill":0.6874,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.9357,"skill":0.9143,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.2076,"skill":0.1822,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8923,"skill":0.8416,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.8316,"skill":0.6724,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.7871,"skill":0.7844,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.8828,"skill":0.8821,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.8406,"skill":0.8375,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.7273,"skill":0.7273,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.2872,"skill":0.2872,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author-submitted full run, not re-timed by us: kit 19ad28e runner (one process, rows strictly serial) with the PR #8 JebadiahEngine in-process on one RTX PRO 6000 (HF Job), bf16 SDPA + fla 0.5.2 Gated DeltaNet kernels, prefix_cache=true (shared system+state prefix once per row, question suffixes from cached KV/DeltaNet state), batch_tokens=32768, max_batch=64 (questions batched inside a row only); torch 2.8.0+cu128, transformers 5.17.0","new_in_edition":true},{"engine":"lavoir","name":"Lavoir","long_name":"MoganAI Lavoir ModernBERT-large","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":395881664,"params_estimated":false,"params_from_name":false,"trained_bytes":1686272004,"model_url":"https://huggingface.co/moganai/lavoir","code_url":"https://github.com/moganai/lavoir","base_model":"answerdotai/ModernBERT-large","base_url":"https://huggingface.co/answerdotai/ModernBERT-large","weights_repo":"moganai/lavoir","served_checkpoint":"4c5eaeb99b2368f1416b9893c7aacc58235aa09f"},"completed":120340,"counts":{"ok":120016,"unsupported":324},"lifted":false,"calibration":{"acc":0.3983,"conf":0.531,"ece":0.1492,"brier":0.7278,"over95":0.0235,"n":35093,"benchmarks":32,"rel":[{"lo":0.0,"w":0.1235,"conf":0.0513,"acc":0.076},{"lo":0.1,"w":0.0391,"conf":0.1438,"acc":0.2767},{"lo":0.2,"w":0.0404,"conf":0.2665,"acc":0.2049},{"lo":0.3,"w":0.0964,"conf":0.3502,"acc":0.3113},{"lo":0.4,"w":0.0943,"conf":0.4522,"acc":0.367},{"lo":0.5,"w":0.191,"conf":0.548,"acc":0.4736},{"lo":0.6,"w":0.1314,"conf":0.6494,"acc":0.4856},{"lo":0.7,"w":0.0976,"conf":0.7483,"acc":0.4891},{"lo":0.8,"w":0.0862,"conf":0.849,"acc":0.5201},{"lo":0.9,"w":0.1001,"conf":0.9559,"acc":0.5835}],"areas":{"knowledge":{"ece":0.1472,"acc":0.3631,"conf":0.5064},"language":{"ece":0.2226,"acc":0.4819,"conf":0.7044},"retrieval":{"ece":0.0571,"acc":0.3325,"conf":0.3084},"tools":{"ece":0.1183,"acc":0.4097,"conf":0.4368},"arts":{"ece":0.1436,"acc":0.371,"conf":0.5077}}},"gaps":{"share":0.0027,"reasons":[{"reason":"instructions longer than its native budget (no truncation allowed)","share":0.0026},{"reason":"prompt longer than its context window (no truncation allowed)","share":0.0}]},"latency":{"median":20.0,"p95":49.5,"mean":25.6},"results":{"1":{"score":0.3949,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":22.2},"2":{"score":0.2765,"jev":0.6535,"metric":"nDCG@10","requests":685,"answered":680,"unsupported":5,"errors":0,"median_ms":null},"3":{"score":0.2826,"jev":0.8834,"metric":"accuracy","requests":508,"answered":506,"unsupported":2,"errors":0,"median_ms":null},"4":{"score":0.2795,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":22.4},"5":{"score":0.0944,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":24.7},"6":{"score":0.4908,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":37.7},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.598,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":21.6},"11":{"score":0.301,"jev":0.7175,"metric":"macro-F1","requests":123,"answered":122,"unsupported":1,"errors":0,"median_ms":null},"12":{"score":0.3686,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":19.6},"20":{"score":0.4994,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":19.4},"21":{"score":0.465,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":17.4},"22":{"score":0.0235,"jev":0.1811,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":62.6},"23":{"score":0.4996,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":19.9},"24":{"score":0.347,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":19.3},"25":{"score":0.2551,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.577,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":17.7},"27":{"score":0.366,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":18.4},"28":{"score":0.5083,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":17.4},"29":{"score":0.4199,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":19.8},"30":{"score":0.2426,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":19.6},"31":{"score":0.0854,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":21.4},"32":{"score":0.4189,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0424,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":38.0},"36":{"score":0.1669,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.2815,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20.0},"38":{"score":0.0391,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":78.4},"39":{"score":0.5894,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4629,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":17.5,"tracks":[{"track":"A · Arabic","score":0.129,"headline":false},{"track":"A · English","score":0.4629,"headline":true},{"track":"C · Arabic pairs","score":0.45,"headline":false},{"track":"C · English pairs","score":0.45,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3513,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":19.6},"42":{"score":0.4869,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":20.3},"43":{"score":0.3947,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5036,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":19.6},"45":{"score":0.1597,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.3623,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":20.7},"50":{"score":0.3484,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":21.2},"56":{"score":0.5,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":26.0,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.1553,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":19.7,"edition_added":"v0.2","jev":0.827},"58":{"score":0.3505,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":19.5,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.2338,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":20.9,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.3295,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.2471,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":20.9,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5965,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":20.2,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":30.12,"balanced_skill":8.69,"breadth_skill":8.24},"frozen_scores":{"balanced_raw":25.14,"balanced_skill":3.81,"breadth_skill":3.57},"categories":[{"id":"knowledge","raw":0.2585,"skill":0.0349,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.3803,"skill":0.1037,"coverage":0.9992,"pending":0,"n":10},{"id":"retrieval","raw":0.3071,"skill":0.12,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.2444,"skill":0.1302,"coverage":0.9978,"pending":0,"n":5},{"id":"arts","raw":0.2988,"skill":0.0323,"coverage":0.9999,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.3949,"skill":0.1832,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.2745,"skill":0.1622,"coverage":0.9927,"random":0.1341,"tracks":[]},"6":{"raw":0.4908,"skill":0.0,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.43,"headline":false},{"track":"RouterBench-5shot","score":0.5517,"headline":false}]},"11":{"raw":0.2983,"skill":0.0,"coverage":0.9919,"random":0.3085,"tracks":[]},"20":{"raw":0.4947,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.465,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0237,"skill":0.0161,"coverage":0.9995,"random":0.0078,"tracks":[]},"23":{"raw":0.4899,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3517,"skill":0.1356,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2551,"skill":0.0068,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.2426,"skill":0.0822,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.3139,"headline":false},{"track":"GSM8K-10choice","score":0.1713,"headline":false}]},"31":{"raw":0.0854,"skill":0.0039,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.1669,"skill":0.0575,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.2815,"skill":0.0988,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.4629,"skill":0.309,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.129,"headline":false},{"track":"A · English","score":0.4629,"headline":true},{"track":"C · Arabic pairs","score":0.45,"headline":false},{"track":"C · English pairs","score":0.45,"headline":false}]},"41":{"raw":0.3513,"skill":0.027,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3947,"skill":0.0397,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.5036,"skill":0.0072,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3484,"skill":0.0544,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5,"skill":0.0,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.1553,"skill":0.0499,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.3505,"skill":0.0586,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.2338,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.3295,"skill":0.1619,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.2471,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5965,"skill":0.193,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.4189,"skill":0.0762,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0424,"skill":0.0297,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1597,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.3686,"skill":0.0542,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.5083,"skill":0.0166,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.4199,"skill":0.2265,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0391,"skill":0.0084,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.5894,"skill":0.3961,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.4869,"skill":0.0018,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.2795,"skill":0.2702,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.0944,"skill":0.0889,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.2815,"skill":0.2676,"coverage":0.9961,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.7594,"pending":0.24,"latency_rows":749,"latency_path":"author-submitted LavoirEngine (di_lavoir_engine.py, native=true) in-process on one RTX PRO 6000 Space (rtx-13): ModernBERT-large encoder with SDPA attention (pinned by lavoir/model.py), bf16 autocast, a row's questions length-sorted and batched (64 questions / 65,536 padded tokens per forward); torch 2.14.0+cu130, transformers 5.17.0, laya 0.3.11, lavoir af79b0a","new_in_edition":true},{"engine":"jpt-0.8b","name":"JPT-0.8B","long_name":"JPT-0.8B Qwen3.5-0.8B LoRA","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":873438784,"params_estimated":false,"params_from_name":false,"trained_bytes":1706035272,"model_url":"https://huggingface.co/kirp/jpt-0.8b","code_url":"https://github.com/tic-top/llm2jev","base_model":"Qwen/Qwen3.5-0.8B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B-Base","weights_repo":"kirp/jpt-0.8b","served_checkpoint":"5921ae03fbaff98fcbe781c1ff3f1200dc897fd1"},"completed":120340,"counts":{"ok":120340},"lifted":false,"calibration":{"acc":0.4947,"conf":0.5626,"ece":0.0688,"brier":0.6121,"over95":0.0035,"n":35093,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0641,"conf":0.0755,"acc":0.0506},{"lo":0.1,"w":0.0356,"conf":0.1462,"acc":0.1591},{"lo":0.2,"w":0.0491,"conf":0.257,"acc":0.2267},{"lo":0.3,"w":0.0922,"conf":0.3546,"acc":0.3448},{"lo":0.4,"w":0.1158,"conf":0.4482,"acc":0.3676},{"lo":0.5,"w":0.1915,"conf":0.5457,"acc":0.504},{"lo":0.6,"w":0.1441,"conf":0.6459,"acc":0.5451},{"lo":0.7,"w":0.1183,"conf":0.7474,"acc":0.6251},{"lo":0.8,"w":0.1019,"conf":0.8508,"acc":0.7403},{"lo":0.9,"w":0.0875,"conf":0.9452,"acc":0.868}],"areas":{"knowledge":{"ece":0.072,"acc":0.4448,"conf":0.5167},"language":{"ece":0.0734,"acc":0.609,"conf":0.6798},"retrieval":{"ece":0.0916,"acc":0.3913,"conf":0.459},"tools":{"ece":0.0472,"acc":0.6257,"conf":0.6647},"arts":{"ece":0.0766,"acc":0.403,"conf":0.4739}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":117.5,"p95":341.2,"mean":118.6},"results":{"1":{"score":0.7674,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":197.5},"2":{"score":0.5285,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":424.1},"3":{"score":0.3012,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.3541,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":119.0},"5":{"score":0.1046,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":122.6},"6":{"score":0.6413,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":271.2},"9":{"score":0,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4914,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":122.4},"11":{"score":0.7274,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.3856,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":114.4},"20":{"score":0.5006,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":44.0},"21":{"score":0.5221,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":113.6},"22":{"score":0.024,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":174.6},"23":{"score":0.5288,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":115.0},"24":{"score":0.5017,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":42.5},"25":{"score":0.301,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.8481,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":41.3},"27":{"score":0.6809,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":40.8},"28":{"score":0.532,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":41.4},"29":{"score":0.486,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":114.5},"30":{"score":0.3442,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":115.4},"31":{"score":0.0846,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":117.7},"32":{"score":0.4761,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0394,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":291.5},"36":{"score":0.3277,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.301,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":119.9},"38":{"score":0.0732,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":460.2},"39":{"score":0.7806,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2655,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":113.6,"tracks":[{"track":"A · Arabic","score":0.1504,"headline":false},{"track":"A · English","score":0.2655,"headline":true},{"track":"C · Arabic pairs","score":0.485,"headline":false},{"track":"C · English pairs","score":0.395,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4399,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":43.0},"42":{"score":0.6283,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":119.0},"43":{"score":0.3386,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.544,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":41.9},"45":{"score":0.1717,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2355,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":119.8},"50":{"score":0.3944,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":120.9},"56":{"score":0.5205,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":284.0,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.2795,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":44.1,"edition_added":"v0.2","jev":0.827},"58":{"score":0.4567,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":43.9,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.4995,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":120.8,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.4015,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.5534,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":120.0,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.5485,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":118.2,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":38.65,"balanced_skill":19.22,"breadth_skill":17.78},"frozen_scores":{"balanced_raw":32.64,"balanced_skill":13.65,"breadth_skill":11.41},"categories":[{"id":"knowledge","raw":0.3035,"skill":0.0988,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.4861,"skill":0.2304,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.3506,"skill":0.164,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.4278,"skill":0.3623,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.34,"skill":0.08,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.7674,"skill":0.686,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.5285,"skill":0.4555,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.6413,"skill":0.1574,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.6057,"headline":false},{"track":"RouterBench-5shot","score":0.677,"headline":false}]},"11":{"raw":0.7274,"skill":0.6058,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.4996,"skill":0.0,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5221,"skill":0.0441,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.025,"skill":0.0174,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.536,"skill":0.0721,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5137,"skill":0.3517,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.301,"skill":0.068,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.3442,"skill":0.2072,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.423,"headline":false},{"track":"GSM8K-10choice","score":0.2654,"headline":false}]},"31":{"raw":0.0846,"skill":0.003,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.3277,"skill":0.2394,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.301,"skill":0.1233,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.2655,"skill":0.0551,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1504,"headline":false},{"track":"A · English","score":0.2655,"headline":true},{"track":"C · Arabic pairs","score":0.485,"headline":false},{"track":"C · English pairs","score":0.395,"headline":false}]},"41":{"raw":0.4399,"skill":0.1599,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.3386,"skill":0.0,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.544,"skill":0.088,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.3944,"skill":0.1211,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.5205,"skill":0.041,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.2795,"skill":0.1896,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.4567,"skill":0.2125,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.4995,"skill":0.3327,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.4015,"skill":0.2519,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.5534,"skill":0.074,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.5485,"skill":0.097,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.4761,"skill":0.1671,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.0394,"skill":0.0266,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1717,"skill":0.0091,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.3856,"skill":0.0797,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.532,"skill":0.064,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.486,"skill":0.3147,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.0732,"skill":0.0436,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.7806,"skill":0.6773,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.6283,"skill":0.2768,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.3541,"skill":0.3458,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.1046,"skill":0.0992,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.3012,"skill":0.2877,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.058,"skill":0.058,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"llm2jev 0.6.1 (chat prompt, thinking off, T=1.140) over SGLang 0.5.9 with the card flags (--context-length 32768 --mamba-scheduler-strategy extra_buffer; radix cache and CUDA graphs on) and --attention-backend triton (required by SGLang 0.5.9 for Qwen3.5 hybrid GDN on Blackwell); all questions of a row in one /generate call; one sequential client; one RTX PRO 6000 Space; venv torch 2.9.1+cu128, transformers 4.57.1, nvidia-cudnn-cu12 9.16.0.29 (SGLang start-up requirement)","new_in_edition":true},{"engine":"jpt-4b","name":"JPT-4B","long_name":"JPT-4B Qwen3.5-4B LoRA","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":9078627496,"model_url":"https://huggingface.co/kirp/jpt-4b","code_url":"https://github.com/tic-top/llm2jev","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"kirp/jpt-4b","served_checkpoint":"277ae37b7a6398a8ec3b86712ac16c1991ab6ef1"},"completed":120340,"counts":{"ok":120340},"lifted":false,"calibration":{"acc":0.6649,"conf":0.7443,"ece":0.0793,"brier":0.4478,"over95":0.0148,"n":35093,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0103,"conf":0.0836,"acc":0.0688},{"lo":0.1,"w":0.0316,"conf":0.147,"acc":0.1099},{"lo":0.2,"w":0.0301,"conf":0.254,"acc":0.1719},{"lo":0.3,"w":0.045,"conf":0.352,"acc":0.3049},{"lo":0.4,"w":0.0525,"conf":0.4532,"acc":0.3683},{"lo":0.5,"w":0.0974,"conf":0.5512,"acc":0.4766},{"lo":0.6,"w":0.1008,"conf":0.6541,"acc":0.5386},{"lo":0.7,"w":0.0966,"conf":0.7547,"acc":0.6209},{"lo":0.8,"w":0.1584,"conf":0.8532,"acc":0.7374},{"lo":0.9,"w":0.3773,"conf":0.9642,"acc":0.9144}],"areas":{"knowledge":{"ece":0.0812,"acc":0.5976,"conf":0.6789},"language":{"ece":0.0478,"acc":0.7932,"conf":0.8404},"retrieval":{"ece":0.1172,"acc":0.6371,"conf":0.749},"tools":{"ece":0.0628,"acc":0.8495,"conf":0.8852},"arts":{"ece":0.133,"acc":0.4937,"conf":0.6267}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":139.1,"p95":405.7,"mean":154.9},"results":{"1":{"score":0.9156,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":247.4},"2":{"score":0.6212,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":617.7},"3":{"score":0.687,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7212,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":151.6},"5":{"score":0.6454,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":169.4},"6":{"score":0.7994,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":363.0},"9":{"score":0.125,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6048,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":164.5},"11":{"score":0.7966,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6199,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":132.9},"20":{"score":0.8048,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":58.4},"21":{"score":0.5974,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":131.9},"22":{"score":0.0165,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":347.8},"23":{"score":0.6,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":146.7},"24":{"score":0.7382,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":58.6},"25":{"score":0.4286,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9777,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":57.8},"27":{"score":0.9283,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":57.8},"28":{"score":0.7127,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":58.9},"29":{"score":0.8103,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":134.4},"30":{"score":0.7278,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":135.5},"31":{"score":0.1384,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":149.5},"32":{"score":0.5559,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2176,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":399.6},"36":{"score":0.4215,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4639,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":142.5},"38":{"score":0.2003,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":597.4},"39":{"score":0.9191,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5175,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":134.6,"tracks":[{"track":"A · Arabic","score":0.2694,"headline":false},{"track":"A · English","score":0.5175,"headline":true},{"track":"C · Arabic pairs","score":0.615,"headline":false},{"track":"C · English pairs","score":0.85,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.6343,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":60.1},"42":{"score":0.7846,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":143.8},"43":{"score":0.5088,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6496,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":58.5},"45":{"score":0.1138,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.1888,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":148.4},"50":{"score":0.389,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":151.5},"56":{"score":0.689,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":352.5,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.4889,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":132.9,"edition_added":"v0.2","jev":0.827},"58":{"score":0.6307,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":77.9,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.6405,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":148.3,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.5739,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.7418,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":155.4,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.6085,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":142.3,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":56.53,"balanced_skill":43.04,"breadth_skill":41.62},"frozen_scores":{"balanced_raw":43.25,"balanced_skill":29.61,"breadth_skill":24.8},"categories":[{"id":"knowledge","raw":0.4437,"skill":0.2872,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.676,"skill":0.5247,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.592,"skill":0.4501,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.6129,"skill":0.572,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4529,"skill":0.2584,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9156,"skill":0.886,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6212,"skill":0.5625,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7995,"skill":0.5272,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7859,"headline":false},{"track":"RouterBench-5shot","score":0.813,"headline":false}]},"11":{"raw":0.7966,"skill":0.7058,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.8055,"skill":0.611,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5974,"skill":0.1948,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0149,"skill":0.0072,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5865,"skill":0.173,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7511,"skill":0.6681,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4286,"skill":0.2381,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.7278,"skill":0.6657,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.7134,"headline":false},{"track":"GSM8K-10choice","score":0.7422,"headline":false}]},"31":{"raw":0.1384,"skill":0.0616,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4215,"skill":0.3456,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.4639,"skill":0.3276,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.5175,"skill":0.3792,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2694,"headline":false},{"track":"A · English","score":0.5175,"headline":true},{"track":"C · Arabic pairs","score":0.615,"headline":false},{"track":"C · English pairs","score":0.85,"headline":false}]},"41":{"raw":0.6343,"skill":0.4514,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.5088,"skill":0.2206,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6496,"skill":0.2992,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.389,"skill":0.1133,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.689,"skill":0.378,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.4889,"skill":0.4251,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6307,"skill":0.4647,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.6405,"skill":0.5207,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.5739,"skill":0.4674,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.7418,"skill":0.4646,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.6085,"skill":0.217,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5559,"skill":0.294,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2176,"skill":0.2072,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1138,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.6199,"skill":0.4306,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.7127,"skill":0.4254,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.8103,"skill":0.7471,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.2003,"skill":0.1747,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.9191,"skill":0.881,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7846,"skill":0.5809,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.7212,"skill":0.7176,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.6454,"skill":0.6433,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.687,"skill":0.681,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.125,"skill":0.125,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.2448,"skill":0.2448,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"llm2jev 0.6.1 (chat prompt, thinking off, T=1.036) over SGLang 0.5.9 with the card flags (--context-length 32768 --mamba-scheduler-strategy extra_buffer; radix cache and CUDA graphs on) and --attention-backend triton (required by SGLang 0.5.9 for Qwen3.5 hybrid GDN on Blackwell); all questions of a row in one /generate call; one sequential client; one RTX PRO 6000 Space; venv torch 2.9.1+cu128, transformers 4.57.1, nvidia-cudnn-cu12 9.16.0.29 (SGLang start-up requirement)","new_in_edition":true},{"engine":"jpt-9b","name":"JPT-9B","long_name":"JPT-9B Qwen3.5-9B LoRA","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":9653104368,"params_estimated":false,"params_from_name":false,"trained_bytes":18819728184,"model_url":"https://huggingface.co/kirp/jpt-9b","code_url":"https://github.com/tic-top/llm2jev","base_model":"Qwen/Qwen3.5-9B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B-Base","weights_repo":"kirp/jpt-9b","served_checkpoint":"7114b0c3d9bea6b82dfa2d0691e8d5562cd26d4e"},"completed":120340,"counts":{"ok":120340},"lifted":false,"calibration":{"acc":0.671,"conf":0.7482,"ece":0.0773,"brier":0.4346,"over95":0.0176,"n":35093,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0283,"conf":0.0775,"acc":0.045},{"lo":0.1,"w":0.0278,"conf":0.1411,"acc":0.116},{"lo":0.2,"w":0.0188,"conf":0.255,"acc":0.1374},{"lo":0.3,"w":0.0319,"conf":0.3484,"acc":0.3028},{"lo":0.4,"w":0.0543,"conf":0.4553,"acc":0.3276},{"lo":0.5,"w":0.0939,"conf":0.5475,"acc":0.4734},{"lo":0.6,"w":0.0999,"conf":0.6509,"acc":0.5436},{"lo":0.7,"w":0.1056,"conf":0.755,"acc":0.6422},{"lo":0.8,"w":0.1439,"conf":0.8559,"acc":0.7612},{"lo":0.9,"w":0.3956,"conf":0.966,"acc":0.9107}],"areas":{"knowledge":{"ece":0.0973,"acc":0.6052,"conf":0.7017},"language":{"ece":0.0282,"acc":0.8225,"conf":0.8469},"retrieval":{"ece":0.185,"acc":0.561,"conf":0.745},"tools":{"ece":0.0348,"acc":0.8767,"conf":0.8991},"arts":{"ece":0.0776,"acc":0.5074,"conf":0.585}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":148.6,"p95":461.9,"mean":178.2},"results":{"1":{"score":0.9486,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":358.3},"2":{"score":0.6172,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":783.3},"3":{"score":0.7874,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.6543,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":178.6},"5":{"score":0.362,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":202.0},"6":{"score":0.8007,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":417.1},"9":{"score":0.4318,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5299,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":194.1},"11":{"score":0.822,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6659,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":141.2},"20":{"score":0.8608,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":68.9},"21":{"score":0.5974,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":136.7},"22":{"score":0.047,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":450.0},"23":{"score":0.5752,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":166.0},"24":{"score":0.7794,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":69.0},"25":{"score":0.4133,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9781,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":68.0},"27":{"score":0.9471,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":65.2},"28":{"score":0.7459,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":65.2},"29":{"score":0.859,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":142.3},"30":{"score":0.7472,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":142.0},"31":{"score":0.149,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":177.2},"32":{"score":0.5479,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2358,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":421.5},"36":{"score":0.469,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5264,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":153.4},"38":{"score":0.2278,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":798.5},"39":{"score":0.9317,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5639,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":140.1,"tracks":[{"track":"A · Arabic","score":0.2031,"headline":false},{"track":"A · English","score":0.5639,"headline":true},{"track":"C · Arabic pairs","score":0.67,"headline":false},{"track":"C · English pairs","score":0.925,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.6443,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":71.6},"42":{"score":0.7899,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":168.1},"43":{"score":0.5561,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.669,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":66.8},"45":{"score":0.1218,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.1814,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":172.8},"50":{"score":0.4021,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":179.0},"56":{"score":0.7605,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":381.7,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.5472,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":109.2,"edition_added":"v0.2","jev":0.827},"58":{"score":0.694,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":141.0,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.6796,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":180.8,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.6212,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.7619,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":168.7,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.6635,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":155.9,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":59.23,"balanced_skill":46.89,"breadth_skill":45.2},"frozen_scores":{"balanced_raw":44.84,"balanced_skill":31.94,"breadth_skill":26.62},"categories":[{"id":"knowledge","raw":0.4663,"skill":0.317,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.7044,"skill":0.5672,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.5656,"skill":0.4459,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.7059,"skill":0.6702,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.4738,"skill":0.2856,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9486,"skill":0.9307,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6172,"skill":0.5579,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.8007,"skill":0.5299,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7881,"headline":false},{"track":"RouterBench-5shot","score":0.8133,"headline":false}]},"11":{"raw":0.822,"skill":0.7425,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.8621,"skill":0.7243,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5974,"skill":0.1948,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0428,"skill":0.0353,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.5568,"skill":0.1137,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7954,"skill":0.7272,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4133,"skill":0.2177,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.7472,"skill":0.6923,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.7589,"headline":false},{"track":"GSM8K-10choice","score":0.7354,"headline":false}]},"31":{"raw":0.149,"skill":0.0731,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.469,"skill":0.3992,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.5264,"skill":0.406,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.5639,"skill":0.4389,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2031,"headline":false},{"track":"A · English","score":0.5639,"headline":true},{"track":"C · Arabic pairs","score":0.67,"headline":false},{"track":"C · English pairs","score":0.925,"headline":false}]},"41":{"raw":0.6443,"skill":0.4664,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.5561,"skill":0.2958,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.669,"skill":0.338,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4021,"skill":0.1323,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.7605,"skill":0.521,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.5472,"skill":0.4907,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.694,"skill":0.5565,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.6796,"skill":0.5728,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.6212,"skill":0.5265,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.7619,"skill":0.5063,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.6635,"skill":0.327,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5479,"skill":0.2812,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.2358,"skill":0.2257,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1218,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.6659,"skill":0.4996,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.7459,"skill":0.4918,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.859,"skill":0.812,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.2278,"skill":0.2031,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.9317,"skill":0.8995,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7899,"skill":0.5912,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.6543,"skill":0.6499,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.362,"skill":0.3581,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.7874,"skill":0.7833,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.4318,"skill":0.4318,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.2744,"skill":0.2744,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"llm2jev 0.6.1 (chat prompt, thinking off, T=1.087) over SGLang 0.5.9 with the card flags (--context-length 32768 --mamba-scheduler-strategy extra_buffer; radix cache and CUDA graphs on) and --attention-backend triton (required by SGLang 0.5.9 for Qwen3.5 hybrid GDN on Blackwell); all questions of a row in one /generate call; one sequential client; one RTX PRO 6000 Space; venv torch 2.9.1+cu128, transformers 4.57.1, nvidia-cudnn-cu12 9.16.0.29 (SGLang start-up requirement)","new_in_edition":true},{"engine":"eikos-27b-fp8","name":"Eikos-27B","long_name":"Eikos-27B-FP8 Qwen3.8-27B LoRA","variant":"FP8","meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":31220328864,"model_url":"https://huggingface.co/caiovicentino1/Eikos-27B-FP8","code_url":"https://github.com/caiovicentino/eikos","base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":"caiovicentino1/Eikos-27B-FP8","served_checkpoint":null},"completed":120340,"counts":{"ok":120340},"lifted":false,"calibration":{"acc":0.7053,"conf":0.7428,"ece":0.0574,"brier":0.4009,"over95":0.0005,"n":35093,"benchmarks":32,"rel":[{"lo":0.0,"w":0.002,"conf":0.0848,"acc":0.0195},{"lo":0.1,"w":0.0225,"conf":0.1546,"acc":0.1296},{"lo":0.2,"w":0.0215,"conf":0.2515,"acc":0.1859},{"lo":0.3,"w":0.035,"conf":0.3548,"acc":0.2158},{"lo":0.4,"w":0.0572,"conf":0.4538,"acc":0.363},{"lo":0.5,"w":0.095,"conf":0.5503,"acc":0.4945},{"lo":0.6,"w":0.1088,"conf":0.6514,"acc":0.6061},{"lo":0.7,"w":0.1394,"conf":0.7541,"acc":0.6852},{"lo":0.8,"w":0.2426,"conf":0.8541,"acc":0.7904},{"lo":0.9,"w":0.2762,"conf":0.9411,"acc":0.9773}],"areas":{"knowledge":{"ece":0.0824,"acc":0.6532,"conf":0.7185},"language":{"ece":0.0796,"acc":0.7434,"conf":0.794},"retrieval":{"ece":0.0263,"acc":0.7369,"conf":0.7158},"tools":{"ece":0.0521,"acc":0.936,"conf":0.9},"arts":{"ece":0.0548,"acc":0.5878,"conf":0.6311}}},"gaps":{"share":0.0,"reasons":[]},"latency":{"median":129.7,"p95":509.8,"mean":213.2},"results":{"1":{"score":0.9628,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":229.9},"2":{"score":0.6675,"jev":0.6528,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1149.7},"3":{"score":0.8602,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7902,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":229.9},"5":{"score":0.8769,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":339.9},"6":{"score":0.7971,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":479.8},"9":{"score":0.5114,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6686,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":200.0},"11":{"score":0.804,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7559,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":119.8},"20":{"score":0.9374,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":119.9},"21":{"score":0.61,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":109.9},"22":{"score":0.2865,"jev":0.181,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":764.7},"23":{"score":0.6518,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":139.8},"24":{"score":0.8416,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":119.9},"25":{"score":0.5051,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9907,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":119.8},"27":{"score":0.9693,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":119.9},"28":{"score":0.8406,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":109.9},"29":{"score":0.9296,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":119.9},"30":{"score":0.7085,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":119.9},"31":{"score":0.1716,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":209.8},"32":{"score":0.609,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.283,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":359.7},"36":{"score":0.4832,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5545,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":129.9},"38":{"score":0.224,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1200.0},"39":{"score":0.0,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null,"wrong_rows":979,"contamination":"The whole FinEntity benchmark counts as wrong: its full test set is in the declared training data"},"40":{"score":0.5744,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":109.9,"tracks":[{"track":"A · Arabic","score":0.4169,"headline":false},{"track":"A · English","score":0.5744,"headline":true},{"track":"C · Arabic pairs","score":0.755,"headline":false},{"track":"C · English pairs","score":0.975,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.7134,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":119.9},"42":{"score":0.8409,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":139.5},"43":{"score":0.6807,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.7512,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":119.8},"45":{"score":0.0978,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.1709,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":179.2},"50":{"score":0.4362,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":199.6},"56":{"score":0.6385,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":319.8,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.6095,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":119.9,"edition_added":"v0.2","jev":0.827},"58":{"score":0.7894,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":119.9,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.7924,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":159.9,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.7102,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.741,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":164.9,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.737,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":129.7,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":63.64,"balanced_skill":53.13,"breadth_skill":52.02},"frozen_scores":{"balanced_raw":48.97,"balanced_skill":37.99,"breadth_skill":31.68},"categories":[{"id":"knowledge","raw":0.5191,"skill":0.3991,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.6537,"skill":0.5432,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.6834,"skill":0.5586,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.7702,"skill":0.7443,"coverage":1.0,"pending":0,"n":5},{"id":"arts","raw":0.5561,"skill":0.3979,"coverage":1.0,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9628,"skill":0.9498,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6675,"skill":0.6161,"coverage":1,"random":0.1341,"tracks":[]},"6":{"raw":0.7971,"skill":0.5211,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.785,"headline":false},{"track":"RouterBench-5shot","score":0.8092,"headline":false}]},"11":{"raw":0.804,"skill":0.7165,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.94,"skill":0.8799,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.61,"skill":0.2199,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.2845,"skill":0.2789,"coverage":1,"random":0.0078,"tracks":[]},"23":{"raw":0.6434,"skill":0.2869,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8473,"skill":0.7965,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5051,"skill":0.3401,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.7085,"skill":0.6497,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.7619,"headline":false},{"track":"GSM8K-10choice","score":0.655,"headline":false}]},"31":{"raw":0.1716,"skill":0.0977,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4832,"skill":0.4154,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.5545,"skill":0.4412,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.5744,"skill":0.4525,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4169,"headline":false},{"track":"A · English","score":0.5744,"headline":true},{"track":"C · Arabic pairs","score":0.755,"headline":false},{"track":"C · English pairs","score":0.975,"headline":false}]},"41":{"raw":0.7134,"skill":0.5701,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.6807,"skill":0.4934,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.7512,"skill":0.5024,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4362,"skill":0.1817,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.6385,"skill":0.277,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.6095,"skill":0.5608,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.7894,"skill":0.6947,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.7924,"skill":0.7232,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.7102,"skill":0.6377,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.741,"skill":0.463,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.737,"skill":0.474,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.609,"skill":0.3784,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.283,"skill":0.2735,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.0978,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.7559,"skill":0.6344,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.8406,"skill":0.6812,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.9296,"skill":0.9061,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.224,"skill":0.1992,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.0,"skill":0.0,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.8409,"skill":0.6905,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.7902,"skill":0.7875,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.8769,"skill":0.8762,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.8602,"skill":0.8575,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.5114,"skill":0.5114,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.3164,"skill":0.3164,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"model repo serve.py 1.2 (caiovicentino1/Eikos-27B-FP8@e40a92f, letter readout, calib.json T=1, max_one_pass 160) over vLLM 0.30.0 with the submission's launch flags (serve_vllm.sh flags, --max-model-len 65536, --enable-prefix-caching --mamba-cache-mode align, --max-num-seqs 64, gpu util 0.85); one sequential client; one RTX PRO 6000 Space; venv torch 2.13.0+cu130, transformers 5.17.0 (fleet/eikos-27b-fp8-freeze.txt)","contamination":[{"catalog_id":39,"benchmark":"FinEntity","wrong":979,"flag":"The whole FinEntity benchmark counts as wrong: its full test set is in the declared training data"}],"new_in_edition":true},{"engine":"jet-v6.2","name":"Jet v6.2","long_name":"Jet v6.2 Qwen3.5-4B LoRA","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":8411558448,"model_url":"https://huggingface.co/michaljach/jet","code_url":"https://github.com/michaljach/jet","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"michaljach/jet","served_checkpoint":null},"completed":120340,"counts":{"ok":120097,"unsupported":243},"lifted":false,"calibration":{"acc":0.6598,"conf":0.8008,"ece":0.1411,"brier":0.4793,"over95":0.0192,"n":34838,"benchmarks":32,"rel":[{"lo":0.0,"w":0.0069,"conf":0.0865,"acc":0.0387},{"lo":0.1,"w":0.0233,"conf":0.1455,"acc":0.0637},{"lo":0.2,"w":0.0142,"conf":0.2521,"acc":0.0721},{"lo":0.3,"w":0.0204,"conf":0.355,"acc":0.1701},{"lo":0.4,"w":0.0354,"conf":0.4541,"acc":0.2921},{"lo":0.5,"w":0.0636,"conf":0.5482,"acc":0.4162},{"lo":0.6,"w":0.0735,"conf":0.6478,"acc":0.4509},{"lo":0.7,"w":0.1029,"conf":0.7544,"acc":0.5354},{"lo":0.8,"w":0.1831,"conf":0.8571,"acc":0.6289},{"lo":0.9,"w":0.4768,"conf":0.95,"acc":0.867}],"areas":{"knowledge":{"ece":0.1572,"acc":0.6046,"conf":0.7618},"language":{"ece":0.1097,"acc":0.7396,"conf":0.8493},"retrieval":{"ece":0.1205,"acc":0.7372,"conf":0.8559},"tools":{"ece":0.1504,"acc":0.7814,"conf":0.9288},"arts":{"ece":0.1761,"acc":0.5025,"conf":0.6786}}},"gaps":{"share":0.002,"reasons":[{"reason":"Complete prompt has 26996 tokens; declared limit 16384","share":0.0001},{"reason":"Complete prompt has 26999 tokens; declared limit 16384","share":0.0},{"reason":"Complete prompt has 27017 tokens; declared limit 16384","share":0.0}]},"latency":{"median":46.8,"p95":371.1,"mean":167.9},"results":{"1":{"score":0.9191,"jev":0.9575,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":95.9},"2":{"score":0.6377,"jev":0.653,"metric":"nDCG@10","requests":685,"answered":684,"unsupported":1,"errors":0,"median_ms":1460.7},"3":{"score":0.5433,"jev":0.8819,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7518,"jev":0.7974,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":49.5,"wrong_rows":2,"contamination":"2 BANKING77 test items found in Jet's training data count as wrong"},"5":{"score":0.9107,"jev":0.8927,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":75.2,"wrong_rows":3,"contamination":"3 CLINC150 test items found in Jet's training data count as wrong"},"6":{"score":0.7985,"jev":0.7992,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":210.2},"9":{"score":0.2841,"jev":0.5227,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.646,"jev":0.4295,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":74.3},"11":{"score":0.7679,"jev":0.7169,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6241,"jev":0.7479,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":45.6},"20":{"score":0.8292,"jev":0.906,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":45.7},"21":{"score":0.5986,"jev":0.6187,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":46.0},"22":{"score":0.024,"jev":0.1811,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":316.2},"23":{"score":0.6422,"jev":0.6474,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":47.0},"24":{"score":0.7348,"jev":0.9173,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":45.9},"25":{"score":0.4439,"jev":0.7857,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.979,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":45.6},"27":{"score":0.9206,"jev":0.9778,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":45.6},"28":{"score":0.7064,"jev":0.9195,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":46.0},"29":{"score":0.928,"jev":0.9452,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":46.1},"30":{"score":0.7002,"jev":0.7987,"metric":"accuracy","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":45.7},"31":{"score":0.1254,"jev":0.1722,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":49.4},"32":{"score":0.5652,"jev":0.6609,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3158,"jev":0.2642,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":370.6},"36":{"score":0.4278,"jev":0.4752,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.529,"jev":0.5521,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":46.6},"38":{"score":0.1298,"jev":0.2952,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2869.6},"39":{"score":0.8173,"jev":0.8698,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4356,"jev":0.5051,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":46.3,"tracks":[{"track":"A · Arabic","score":0.3389,"headline":false},{"track":"A · English","score":0.4356,"headline":true},{"track":"C · Arabic pairs","score":0.595,"headline":false},{"track":"C · English pairs","score":0.91,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4791,"jev":0.6463,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":45.8},"42":{"score":0.7718,"jev":0.8406,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":47.2},"43":{"score":0.4421,"jev":0.7298,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6582,"jev":0.7264,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":45.8},"45":{"score":0.1018,"jev":0.2036,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"48":{"score":0.2225,"jev":0.1736,"metric":"Brier (lower is better)","requests":10139,"answered":10139,"unsupported":0,"errors":0,"median_ms":48.1},"50":{"score":0.4475,"jev":0.4594,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":48.4},"56":{"score":0.539,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":411.2,"edition_added":"v0.2","jev":0.6255},"57":{"score":0.4848,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":45.9,"edition_added":"v0.2","jev":0.827},"58":{"score":0.6287,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":45.7,"edition_added":"v0.2","jev":0.9292},"62":{"score":0.8677,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":50.4,"edition_added":"v0.2","jev":0.8097},"64":{"score":0.5871,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7008},"59":{"score":0.5132,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":47.8,"edition_added":"v0.2","jev":0.7653},"61":{"score":0.6392,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":47.1,"edition_added":"v0.2","jev":0.7285}},"scores":{"balanced_raw":56.93,"balanced_skill":42.6,"breadth_skill":41.15},"frozen_scores":{"balanced_raw":43.09,"balanced_skill":29.45,"breadth_skill":24.88},"categories":[{"id":"knowledge","raw":0.4443,"skill":0.2867,"coverage":1.0,"pending":0,"n":10},{"id":"language","raw":0.625,"skill":0.4387,"coverage":1.0,"pending":0,"n":10},{"id":"retrieval","raw":0.6387,"skill":0.4817,"coverage":1.0,"pending":0,"n":6},{"id":"tools","raw":0.6562,"skill":0.6294,"coverage":0.9997,"pending":0,"n":5},{"id":"arts","raw":0.451,"skill":0.2702,"coverage":0.9999,"pending":0,"n":7}],"benchmarks":{"1":{"raw":0.9191,"skill":0.8908,"coverage":1,"random":0.2592,"tracks":[]},"2":{"raw":0.6368,"skill":0.5806,"coverage":0.9985,"random":0.1341,"tracks":[]},"6":{"raw":0.7986,"skill":0.5248,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7858,"headline":false},{"track":"RouterBench-5shot","score":0.8113,"headline":false}]},"11":{"raw":0.7679,"skill":0.6643,"coverage":1,"random":0.3085,"tracks":[]},"20":{"raw":0.8298,"skill":0.6597,"coverage":1,"random":0.5,"tracks":[]},"21":{"raw":0.5986,"skill":0.1971,"coverage":1,"random":0.5,"tracks":[]},"22":{"raw":0.0216,"skill":0.0139,"coverage":0.9995,"random":0.0078,"tracks":[]},"23":{"raw":0.6305,"skill":0.2609,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7466,"skill":0.6621,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4439,"skill":0.2585,"coverage":1,"random":0.25,"tracks":[]},"30":{"raw":0.7002,"skill":0.6337,"coverage":1,"random":0.25,"tracks":[{"track":"GSM8K-4choice","score":0.702,"headline":false},{"track":"GSM8K-10choice","score":0.6983,"headline":false}]},"31":{"raw":0.1254,"skill":0.0474,"coverage":1,"random":0.0819,"tracks":[]},"36":{"raw":0.4278,"skill":0.3527,"coverage":1,"random":0.116,"tracks":[]},"37":{"raw":0.529,"skill":0.4092,"coverage":1,"random":0.2027,"tracks":[]},"40":{"raw":0.4356,"skill":0.2739,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3389,"headline":false},{"track":"A · English","score":0.4356,"headline":true},{"track":"C · Arabic pairs","score":0.595,"headline":false},{"track":"C · English pairs","score":0.91,"headline":false}]},"41":{"raw":0.4791,"skill":0.2187,"coverage":1,"random":0.3333,"tracks":[]},"43":{"raw":0.4421,"skill":0.1148,"coverage":1,"random":0.3697,"tracks":[]},"44":{"raw":0.6582,"skill":0.3164,"coverage":1,"random":0.5,"tracks":[]},"50":{"raw":0.4475,"skill":0.1981,"coverage":1,"random":0.311,"tracks":[]},"7":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"16":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"13":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"15":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"17":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"19":{"raw":0.0,"skill":0.0,"coverage":0.0,"random":null,"tracks":[]},"56":{"raw":0.539,"skill":0.078,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"57":{"raw":0.4848,"skill":0.4205,"coverage":1.0,"random":0.1109,"tracks":[],"edition_added":"v0.2"},"58":{"raw":0.6287,"skill":0.4618,"coverage":1.0,"random":0.3101,"tracks":[],"edition_added":"v0.2"},"62":{"raw":0.8677,"skill":0.8236,"coverage":1.0,"random":0.25,"tracks":[],"edition_added":"v0.2"},"64":{"raw":0.5871,"skill":0.4839,"coverage":1.0,"random":0.2,"tracks":[],"edition_added":"v0.2"},"59":{"raw":0.5132,"skill":0.0,"coverage":1.0,"random":0.5177,"tracks":[],"edition_added":"v0.2"},"61":{"raw":0.6392,"skill":0.2784,"coverage":1.0,"random":0.5,"tracks":[],"edition_added":"v0.2"},"32":{"raw":0.5652,"skill":0.3087,"coverage":1.0,"random":0.371,"tracks":[]},"33":{"raw":0.3158,"skill":0.3067,"coverage":1.0,"random":0.0131,"tracks":[]},"45":{"raw":0.1018,"skill":0.0,"coverage":1.0,"random":0.1641,"tracks":[]},"12":{"raw":0.6241,"skill":0.4369,"coverage":1.0,"random":0.3324,"tracks":[]},"28":{"raw":0.7064,"skill":0.4128,"coverage":1.0,"random":0.5,"tracks":[]},"29":{"raw":0.928,"skill":0.904,"coverage":1.0,"random":0.25,"tracks":[]},"38":{"raw":0.1298,"skill":0.102,"coverage":1.0,"random":0.031,"tracks":[]},"39":{"raw":0.8173,"skill":0.7313,"coverage":1.0,"random":0.3201,"tracks":[]},"42":{"raw":0.7718,"skill":0.556,"coverage":1.0,"random":0.486,"tracks":[]},"4":{"raw":0.7518,"skill":0.7486,"coverage":1.0,"random":0.0127,"tracks":[]},"5":{"raw":0.9107,"skill":0.9102,"coverage":1.0,"random":0.006,"tracks":[]},"3":{"raw":0.5433,"skill":0.5345,"coverage":1.0,"random":0.0189,"tracks":[]},"9":{"raw":0.2841,"skill":0.2841,"coverage":1.0,"random":0.0,"tracks":[]},"48":{"raw":0.11,"skill":0.11,"coverage":1.0,"random":0.25,"tracks":[],"index_rule":"vs baseline"}},"coverage":0.7599,"pending":0.24,"latency_rows":750,"latency_path":"kit PR #20 engine submissions/jet/engine.py @21092e8 (sha256 357454d0) unchanged over michaljach/jet@fbc3d2d format.py/runtime.py: BF16, SDPA, FLA chunk gated-delta kernel, PyTorch causal-conv fallback as the runtime specifies, one forward per question, no cache; the only path the published runtime has; venv torch 2.11.0+cu128, transformers 5.17.0, fla 0.5.2, triton 3.6.0 (the model repo requirements.txt); fresh process, cold autotune cache; warm-cache rerun kept as latency-v1-full.rerun (47.1 / 319.0 ms, bit-identical answers)","contamination":[{"catalog_id":4,"benchmark":"BANKING77","wrong":2,"flag":"2 BANKING77 test items found in Jet's training data count as wrong"},{"catalog_id":5,"benchmark":"CLINC150+OOS","wrong":3,"flag":"3 CLINC150 test items found in Jet's training data count as wrong"}],"new_in_edition":true}]}