{"benchmark":"JevBench","revision":"v1.5.0","protocol":"jevbench::v1.5","status":"released","run_kind":"official","source_sha256":"452885de2a84cd5b9ed393d541fd8d6c6a540f9d7f762f9d2df9349383f26173","base_results_sha256":"1cea4d6018da4c7e61132e932452b20ddea5758dcb8714a57a858e078dd6af12","addenda_sources_sha256":{"A1":"2101bff95e6056fdebbcc6cfdb9e86c634ac8fb4a39419389c38bb47732e8c88","A1_protocol":"fadbcd6ab18cc4576deab9f5dd7bc7e0af1eb93d111648719083e4093125b7ef","A2":"eea496b1a549963084db76659fe32d3a67c2e9ca281e1d6a231df9be759343a7","A2_protocol":"1fb818a123eedf68c49295fd860f62b27c79b80fb85953b2a2e86599f861f0f3","base":"1cea4d6018da4c7e61132e932452b20ddea5758dcb8714a57a858e078dd6af12","pricing_disclosure_correction":"1b660648bd498645a0888bd365bb24cfc616c79542862594d7a0077ed1db51fb","pricing_disclosure_normalizer":"2cbc7bd5fdd9c249d526e4269ee492261feafecabd0cc662251437223776f586"},"method_sha256":"c25d3d8b8512e4d93370a9e0c99705d19b2a9389956ca33b8a4bd2b0ec501c07","pricing_addendum_sha256":"2fc44459ef801d0627062f7eefd973df40772e8ac117727479748e4be4c220cc","headline_method_addendum_sha256":"752ddccc4e191c81412631ee3121563421c4b9077cb6c3f2243ed1d1545cfab0","pricing_disclosure_correction_sha256":"1b660648bd498645a0888bd365bb24cfc616c79542862594d7a0077ed1db51fb","pricing_disclosure_normalizer_sha256":"2cbc7bd5fdd9c249d526e4269ee492261feafecabd0cc662251437223776f586","sample":{"open":904,"sealed":720,"total":1624,"published_open":601},"types":{"choice":0.3333333333333333,"noul":0.3333333333333333,"score":0.3333333333333333},"tier_weights":{"easy":0.1,"standard":0.2,"judge":0.3,"hard":0.4},"sealed_share_of_intelligence":0.5,"G_med":5.186627500079243,"G_med_flag_gt10":false,"headline":"A","options":{"A":{"weights":{"intelligence":25,"calibration":25,"speed":25,"cost":25},"intelligence_floor":50},"B":{"weights":{"intelligence":40,"calibration":20,"speed":20,"cost":20},"intelligence_floor":50},"C":{"weights":{"intelligence":25,"calibration":25,"speed":25,"cost":25},"intelligence_floor":60}},"views":["Balanced 33:33:33","Emphasis on Accuracy 60:20:20","Emphasis on Cost 20:20:60","Emphasis on Speed 20:60:20","Intelligence only"],"bootstrap":{"B":2000,"bootstrap_seed":15},"n_ranked":89,"board":{"A":{"order":["cygnet","winnow-12b","jev-1.13.0","jev-omni","decider-4b-v2","semif-qwen3.5-4b","spark-s1-4b-v6","metask-jev-4b","hopper","malkuth-4b","reflex-4b","jev-local","djev","raw-qwen3-4b-instruct-2507","jqv","jevk5-v02","qwen35-9b-jev-data-mix-v2","standardone-8b","ninfer-qwen3.8-flash-next","swanone","raw-qwen3-8b","decider-2b","system-one-sg","system-one-open","kushal-gemma4-31b-it-autoloops","gpt-6-luna-low","gpt-6-luna","jevone","kev-4b","kev-8b","open-alternative-jev","nimble-9b","malkuth-2b","openjev-sglang","decider-35b-a3b","localjev-qwen3.5-4b","open-jev-zefan-9b","decision-2b","gpt-5.6-luna","typecastlm","jev-qwen3.5-9b-base-nvfp4","gemini-3.1-flash-lite","ninfer-qwen3.8-27b","ninfer-qwen3.8-27b-t1.5","instinct","openjev-thinking","djev-thinking","litjev","raw-phi-4-mini","opensourcejev-qwen35-4b-q4km","reflex-27b","open-jev-zefan-2b","gliner2-large","qwen3-reranker-4b","deepseek-flash","simplejev-qwen3.5-0.8b","simplejev-qwen3.8-27b","decision-machine-1","gliner2.5-multi","gliner2","jevact","clm-8b","kev-0.6b","raw-qwen3-0.6b","gliner2.5-small","raw-qwen3-1.7b","mirror","zerank-2","jeff","smalljev","opendecision","bge-reranker-v2-m3","certo","decision-fast","gte-reranker-modernbert-base","kev-0.5b","laya","lev-350m","mghafiri-qwen3.5-0.8b-decision-model","mxbai-rerank-base-v2","needle-3","needle-3-tools","open-jev-deberta-v3-large","open-jev-json-canvas-joshuasp","openjev-verdict","openjev-verdict-1.4","qwen3.8-27b","verdict-small","von-395m"],"leader_wording":"joint leaders (statistical tie)","markers":[{"upper":"cygnet","lower":"winnow-12b","tie":true,"diff_ci95":[-0.46125459419296033,1.2374474782799036],"p_upper_wins":0.8345},{"upper":"winnow-12b","lower":"jev-1.13.0","tie":false,"diff_ci95":[0.2845795431300456,2.0792169124533744],"p_upper_wins":0.995},{"upper":"jev-1.13.0","lower":"jev-omni","tie":true,"diff_ci95":[-0.4111511855486413,1.4641041378552804],"p_upper_wins":0.837},{"upper":"jev-omni","lower":"decider-4b-v2","tie":true,"diff_ci95":[-1.038582204987227,2.199403871434391],"p_upper_wins":0.7245},{"upper":"decider-4b-v2","lower":"semif-qwen3.5-4b","tie":false,"diff_ci95":[0.6211698505362904,10.276819386401833],"p_upper_wins":0.994},{"upper":"semif-qwen3.5-4b","lower":"spark-s1-4b-v6","tie":true,"diff_ci95":[-7.2683777036922965,2.1182629795515853],"p_upper_wins":0.5735},{"upper":"spark-s1-4b-v6","lower":"metask-jev-4b","tie":true,"diff_ci95":[-0.8854612486139711,2.800075681219657],"p_upper_wins":0.8275},{"upper":"metask-jev-4b","lower":"hopper","tie":true,"diff_ci95":[-1.8537605222526754,9.941454259279055],"p_upper_wins":0.5785},{"upper":"hopper","lower":"malkuth-4b","tie":true,"diff_ci95":[-9.382473047380726,2.590058496634971],"p_upper_wins":0.5305},{"upper":"malkuth-4b","lower":"reflex-4b","tie":false,"diff_ci95":[0.2574386792786932,7.041235943079069],"p_upper_wins":0.9905},{"upper":"reflex-4b","lower":"jev-local","tie":true,"diff_ci95":[-6.041152723112148,1.492373069358983],"p_upper_wins":0.4325},{"upper":"jev-local","lower":"djev","tie":true,"diff_ci95":[-0.5525291189700304,2.5296882748744998],"p_upper_wins":0.8925},{"upper":"djev","lower":"raw-qwen3-4b-instruct-2507","tie":true,"diff_ci95":[-0.23307398561861295,4.86568485012425],"p_upper_wins":0.9605},{"upper":"raw-qwen3-4b-instruct-2507","lower":"jqv","tie":true,"diff_ci95":[-2.9153289030224725,10.293731051459305],"p_upper_wins":0.656},{"upper":"jqv","lower":"jevk5-v02","tie":true,"diff_ci95":[-6.493803455404365,10.969443174861983],"p_upper_wins":0.656},{"upper":"jevk5-v02","lower":"qwen35-9b-jev-data-mix-v2","tie":true,"diff_ci95":[-5.0456850000987785,14.787465294682384],"p_upper_wins":0.827},{"upper":"qwen35-9b-jev-data-mix-v2","lower":"standardone-8b","tie":false,"diff_ci95":[4.309767791178995,6.238131399932088],"p_upper_wins":1.0},{"upper":"standardone-8b","lower":"ninfer-qwen3.8-flash-next","tie":true,"diff_ci95":[-0.6416244151701841,1.2262471900961953],"p_upper_wins":0.749},{"upper":"ninfer-qwen3.8-flash-next","lower":"swanone","tie":false,"diff_ci95":[0.39497981009657546,1.4306319415544606],"p_upper_wins":1.0},{"upper":"swanone","lower":"raw-qwen3-8b","tie":true,"diff_ci95":[-0.32051802578132094,9.204043926273682],"p_upper_wins":0.941},{"upper":"raw-qwen3-8b","lower":"decider-2b","tie":true,"diff_ci95":[-9.716705520853559,9.766722902363652],"p_upper_wins":0.513},{"upper":"decider-2b","lower":"system-one-sg","tie":true,"diff_ci95":[-8.46504370655965,10.581308485732498],"p_upper_wins":0.583},{"upper":"system-one-sg","lower":"system-one-open","tie":true,"diff_ci95":[-8.708966141268096,9.288384655250024],"p_upper_wins":0.553},{"upper":"system-one-open","lower":"kushal-gemma4-31b-it-autoloops","tie":true,"diff_ci95":[-6.531864021404802,11.396874568395233],"p_upper_wins":0.6615},{"upper":"kushal-gemma4-31b-it-autoloops","lower":"gpt-6-luna-low","tie":true,"diff_ci95":[-0.4144644818337552,0.32506232006125557],"p_upper_wins":0.439},{"upper":"gpt-6-luna-low","lower":"gpt-6-luna","tie":false,"diff_ci95":[1.5583286196261574,1.8857336022517284],"p_upper_wins":1.0},{"upper":"gpt-6-luna","lower":"jevone","tie":true,"diff_ci95":[-0.03151412517146692,1.3843962177828855],"p_upper_wins":0.967},{"upper":"jevone","lower":"kev-4b","tie":true,"diff_ci95":[-8.509048186025899,10.610625427507967],"p_upper_wins":0.579},{"upper":"kev-4b","lower":"kev-8b","tie":true,"diff_ci95":[-3.7433056342930913,11.397615769700971],"p_upper_wins":0.8385},{"upper":"kev-8b","lower":"open-alternative-jev","tie":true,"diff_ci95":[-8.771759347590297,7.6265715775051195],"p_upper_wins":0.485},{"upper":"open-alternative-jev","lower":"nimble-9b","tie":true,"diff_ci95":[-5.855456245563838,9.74872432947998],"p_upper_wins":0.6465},{"upper":"nimble-9b","lower":"malkuth-2b","tie":true,"diff_ci95":[-7.142707535889598,10.586944134669313],"p_upper_wins":0.662},{"upper":"malkuth-2b","lower":"openjev-sglang","tie":true,"diff_ci95":[-8.006951980847019,10.100145938288167],"p_upper_wins":0.5605},{"upper":"openjev-sglang","lower":"decider-35b-a3b","tie":false,"diff_ci95":[0.9680817033605356,2.051678046459891],"p_upper_wins":1.0},{"upper":"decider-35b-a3b","lower":"localjev-qwen3.5-4b","tie":true,"diff_ci95":[-5.352522919587847,7.400630850160767],"p_upper_wins":0.6905},{"upper":"localjev-qwen3.5-4b","lower":"open-jev-zefan-9b","tie":true,"diff_ci95":[-4.386223433809196,8.569055333061891],"p_upper_wins":0.675},{"upper":"open-jev-zefan-9b","lower":"decision-2b","tie":true,"diff_ci95":[-4.4136769003767755,7.468775417922205],"p_upper_wins":0.738},{"upper":"decision-2b","lower":"gpt-5.6-luna","tie":true,"diff_ci95":[-5.633641938976553,6.477057055322495],"p_upper_wins":0.499},{"upper":"gpt-5.6-luna","lower":"typecastlm","tie":true,"diff_ci95":[-6.241861739411629,6.126494390998944],"p_upper_wins":0.58},{"upper":"typecastlm","lower":"jev-qwen3.5-9b-base-nvfp4","tie":true,"diff_ci95":[-4.540142594145304,8.509187230312083],"p_upper_wins":0.706},{"upper":"jev-qwen3.5-9b-base-nvfp4","lower":"gemini-3.1-flash-lite","tie":true,"diff_ci95":[-4.178309047666805,5.746017130776264],"p_upper_wins":0.565},{"upper":"gemini-3.1-flash-lite","lower":"ninfer-qwen3.8-27b","tie":false,"diff_ci95":[0.6373929781328598,1.1278886019384695],"p_upper_wins":1.0},{"upper":"ninfer-qwen3.8-27b","lower":"ninfer-qwen3.8-27b-t1.5","tie":false,"diff_ci95":[0.14663272307435699,0.39815123666500396],"p_upper_wins":1.0},{"upper":"ninfer-qwen3.8-27b-t1.5","lower":"instinct","tie":true,"diff_ci95":[-0.024202650340603427,0.3015667277965538],"p_upper_wins":0.957},{"upper":"instinct","lower":"openjev-thinking","tie":false,"diff_ci95":[0.10228253132511417,0.5988127921420549],"p_upper_wins":0.9965},{"upper":"openjev-thinking","lower":"djev-thinking","tie":false,"diff_ci95":[0.3544604920141397,0.6977343610857955],"p_upper_wins":1.0},{"upper":"djev-thinking","lower":"litjev","tie":false,"diff_ci95":[0.8933699543947561,1.3761614082369105],"p_upper_wins":1.0},{"upper":"litjev","lower":"raw-phi-4-mini","tie":true,"diff_ci95":[-4.407182033300111,5.911981192696572],"p_upper_wins":0.678},{"upper":"raw-phi-4-mini","lower":"opensourcejev-qwen35-4b-q4km","tie":true,"diff_ci95":[-4.188941906621974,8.152834924030318],"p_upper_wins":0.727},{"upper":"opensourcejev-qwen35-4b-q4km","lower":"reflex-27b","tie":true,"diff_ci95":[-4.086740231332742,5.091597668154737],"p_upper_wins":0.5045},{"upper":"reflex-27b","lower":"open-jev-zefan-2b","tie":false,"diff_ci95":[1.6762420468103854,6.464898360709222],"p_upper_wins":0.9995},{"upper":"open-jev-zefan-2b","lower":"gliner2-large","tie":true,"diff_ci95":[-4.020653839372584,4.574881867858114],"p_upper_wins":0.613},{"upper":"gliner2-large","lower":"qwen3-reranker-4b","tie":true,"diff_ci95":[-2.6235372779379844,5.458934880708517],"p_upper_wins":0.754},{"upper":"qwen3-reranker-4b","lower":"deepseek-flash","tie":true,"diff_ci95":[-2.3971447414709712,4.087291868438116],"p_upper_wins":0.5605},{"upper":"deepseek-flash","lower":"simplejev-qwen3.5-0.8b","tie":true,"diff_ci95":[-0.016820216423917355,4.583231154534746],"p_upper_wins":0.974},{"upper":"simplejev-qwen3.5-0.8b","lower":"simplejev-qwen3.8-27b","tie":true,"diff_ci95":[-1.2913920287617369,3.288144558865908],"p_upper_wins":0.7195},{"upper":"simplejev-qwen3.8-27b","lower":"decision-machine-1","tie":true,"diff_ci95":[-2.043185939748674,1.621607948589032],"p_upper_wins":0.5625},{"upper":"decision-machine-1","lower":"gliner2.5-multi","tie":true,"diff_ci95":[-2.0466439303901143,3.0151451587566616],"p_upper_wins":0.653},{"upper":"gliner2.5-multi","lower":"gliner2","tie":true,"diff_ci95":[-1.8543985948833104,2.732411778791044],"p_upper_wins":0.6605},{"upper":"gliner2","lower":"jevact","tie":true,"diff_ci95":[-1.37742137346713,3.224283034579049],"p_upper_wins":0.756},{"upper":"jevact","lower":"clm-8b","tie":true,"diff_ci95":[-1.702477083526242,1.8005172541805738],"p_upper_wins":0.5205},{"upper":"clm-8b","lower":"kev-0.6b","tie":true,"diff_ci95":[-1.2082033404420878,1.8475007171872655],"p_upper_wins":0.6055},{"upper":"kev-0.6b","lower":"raw-qwen3-0.6b","tie":true,"diff_ci95":[-1.4493108577709668,1.7053664084996474],"p_upper_wins":0.5925},{"upper":"raw-qwen3-0.6b","lower":"gliner2.5-small","tie":true,"diff_ci95":[-1.1563285979099487,1.732142807021534],"p_upper_wins":0.599},{"upper":"gliner2.5-small","lower":"raw-qwen3-1.7b","tie":true,"diff_ci95":[-1.4041473705371919,1.4426281090931599],"p_upper_wins":0.5165},{"upper":"raw-qwen3-1.7b","lower":"mirror","tie":true,"diff_ci95":[-0.31468926986976725,2.0745338529312134],"p_upper_wins":0.892},{"upper":"mirror","lower":"zerank-2","tie":true,"diff_ci95":[-0.27793465343426604,0.755911422270539],"p_upper_wins":0.666},{"upper":"zerank-2","lower":"jeff","tie":true,"diff_ci95":[-0.3853413905470331,0.3522016103947976],"p_upper_wins":0.561},{"upper":"jeff","lower":"smalljev","tie":true,"diff_ci95":[-0.2533892169562256,0.41879249587200007],"p_upper_wins":0.6425},{"upper":"smalljev","lower":"opendecision","tie":true,"diff_ci95":[-0.12045273710115305,0.3475332681246026],"p_upper_wins":0.7875},{"upper":"opendecision","lower":"bge-reranker-v2-m3","tie":true,"diff_ci95":[0.0,0.19294313592439996],"p_upper_wins":0.892},{"upper":"bge-reranker-v2-m3","lower":"certo","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"certo","lower":"decision-fast","tie":true,"diff_ci95":[-0.0001584994708377039,0.0],"p_upper_wins":0.0},{"upper":"decision-fast","lower":"gte-reranker-modernbert-base","tie":true,"diff_ci95":[0.0,0.00015849947083770386],"p_upper_wins":0.0505},{"upper":"gte-reranker-modernbert-base","lower":"kev-0.5b","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"kev-0.5b","lower":"laya","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"laya","lower":"lev-350m","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"lev-350m","lower":"mghafiri-qwen3.5-0.8b-decision-model","tie":true,"diff_ci95":[-0.022833461476708714,0.0],"p_upper_wins":0.0},{"upper":"mghafiri-qwen3.5-0.8b-decision-model","lower":"mxbai-rerank-base-v2","tie":true,"diff_ci95":[0.0,0.022833461476708693],"p_upper_wins":0.475},{"upper":"mxbai-rerank-base-v2","lower":"needle-3","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"needle-3","lower":"needle-3-tools","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"needle-3-tools","lower":"open-jev-deberta-v3-large","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"open-jev-deberta-v3-large","lower":"open-jev-json-canvas-joshuasp","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"open-jev-json-canvas-joshuasp","lower":"openjev-verdict","tie":true,"diff_ci95":[-0.014760916697897619,0.0],"p_upper_wins":0.0},{"upper":"openjev-verdict","lower":"openjev-verdict-1.4","tie":true,"diff_ci95":[0.0,0.014760916697897527],"p_upper_wins":0.325},{"upper":"openjev-verdict-1.4","lower":"qwen3.8-27b","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"qwen3.8-27b","lower":"verdict-small","tie":true,"diff_ci95":[-0.0051515709883764756,0.0],"p_upper_wins":0.0},{"upper":"verdict-small","lower":"von-395m","tie":true,"diff_ci95":[0.0,0.005151570988376465],"p_upper_wins":0.1255}]},"B":{"order":["winnow-12b","cygnet","jev-1.13.0","jev-omni","decider-4b-v2","spark-s1-4b-v6","djev","semif-qwen3.5-4b","metask-jev-4b","malkuth-4b","jev-local","hopper","reflex-4b","raw-qwen3-4b-instruct-2507","jqv","jevk5-v02","qwen35-9b-jev-data-mix-v2","ninfer-qwen3.8-flash-next","swanone","standardone-8b","raw-qwen3-8b","system-one-sg","gpt-6-luna-low","kushal-gemma4-31b-it-autoloops","gpt-6-luna","decider-2b","system-one-open","jevone","kev-4b","kev-8b","nimble-9b","open-alternative-jev","openjev-sglang","decider-35b-a3b","malkuth-2b","open-jev-zefan-9b","gpt-5.6-luna","localjev-qwen3.5-4b","gemini-3.1-flash-lite","ninfer-qwen3.8-27b","decision-2b","openjev-thinking","ninfer-qwen3.8-27b-t1.5","instinct","typecastlm","djev-thinking","jev-qwen3.5-9b-base-nvfp4","litjev","reflex-27b","raw-phi-4-mini","opensourcejev-qwen35-4b-q4km","open-jev-zefan-2b","deepseek-flash","gliner2-large","qwen3-reranker-4b","simplejev-qwen3.8-27b","simplejev-qwen3.5-0.8b","decision-machine-1","gliner2.5-multi","gliner2","clm-8b","jevact","kev-0.6b","raw-qwen3-0.6b","raw-qwen3-1.7b","gliner2.5-small","mirror","zerank-2","jeff","smalljev","opendecision","bge-reranker-v2-m3","certo","decision-fast","gte-reranker-modernbert-base","kev-0.5b","laya","lev-350m","mghafiri-qwen3.5-0.8b-decision-model","mxbai-rerank-base-v2","needle-3","needle-3-tools","open-jev-deberta-v3-large","open-jev-json-canvas-joshuasp","openjev-verdict","openjev-verdict-1.4","qwen3.8-27b","verdict-small","von-395m"],"leader_wording":"joint leaders (statistical tie)","markers":[{"upper":"winnow-12b","lower":"cygnet","tie":true,"diff_ci95":[-0.7084696230272624,1.4585613409064304],"p_upper_wins":0.731},{"upper":"cygnet","lower":"jev-1.13.0","tie":true,"diff_ci95":[-0.06610426174216853,2.3130221909080624],"p_upper_wins":0.9645},{"upper":"jev-1.13.0","lower":"jev-omni","tie":true,"diff_ci95":[-0.5431497817752661,1.9720661401099062],"p_upper_wins":0.851},{"upper":"jev-omni","lower":"decider-4b-v2","tie":false,"diff_ci95":[1.9742789859732,6.3688209916817256],"p_upper_wins":1.0},{"upper":"decider-4b-v2","lower":"spark-s1-4b-v6","tie":true,"diff_ci95":[-2.1946707969666193,2.596028382828773],"p_upper_wins":0.6075},{"upper":"spark-s1-4b-v6","lower":"djev","tie":false,"diff_ci95":[0.3139157941753496,3.873436058185904],"p_upper_wins":0.9885},{"upper":"djev","lower":"semif-qwen3.5-4b","tie":true,"diff_ci95":[-1.5767426990756088,8.620787305020604],"p_upper_wins":0.673},{"upper":"semif-qwen3.5-4b","lower":"metask-jev-4b","tie":true,"diff_ci95":[-6.664966544405195,2.275274677429068],"p_upper_wins":0.5525},{"upper":"metask-jev-4b","lower":"malkuth-4b","tie":true,"diff_ci95":[-1.8071467456760701,2.0510200547009267],"p_upper_wins":0.5855},{"upper":"malkuth-4b","lower":"jev-local","tie":true,"diff_ci95":[-1.8461747180925758,2.909436298771614],"p_upper_wins":0.7015},{"upper":"jev-local","lower":"hopper","tie":true,"diff_ci95":[-2.075070320770288,10.280103386220935],"p_upper_wins":0.6285},{"upper":"hopper","lower":"reflex-4b","tie":true,"diff_ci95":[-8.003191618773977,4.910504591450937],"p_upper_wins":0.599},{"upper":"reflex-4b","lower":"raw-qwen3-4b-instruct-2507","tie":true,"diff_ci95":[-4.527115700264317,4.123040901228487],"p_upper_wins":0.78},{"upper":"raw-qwen3-4b-instruct-2507","lower":"jqv","tie":true,"diff_ci95":[-1.8101263442107425,11.774500744579674],"p_upper_wins":0.819},{"upper":"jqv","lower":"jevk5-v02","tie":true,"diff_ci95":[-5.2238636055052465,12.095254186677929],"p_upper_wins":0.7445},{"upper":"jevk5-v02","lower":"qwen35-9b-jev-data-mix-v2","tie":true,"diff_ci95":[-8.729909910429436,10.644321838901877],"p_upper_wins":0.5755},{"upper":"qwen35-9b-jev-data-mix-v2","lower":"ninfer-qwen3.8-flash-next","tie":false,"diff_ci95":[3.5266856947312375,6.155260792180568],"p_upper_wins":1.0},{"upper":"ninfer-qwen3.8-flash-next","lower":"swanone","tie":true,"diff_ci95":[-0.3282641122399742,1.1334253178856313],"p_upper_wins":0.85},{"upper":"swanone","lower":"standardone-8b","tie":true,"diff_ci95":[-1.0728694554302445,1.427460514019379],"p_upper_wins":0.6055},{"upper":"standardone-8b","lower":"raw-qwen3-8b","tie":false,"diff_ci95":[0.5776097362594756,10.613504124081201],"p_upper_wins":0.9945},{"upper":"raw-qwen3-8b","lower":"system-one-sg","tie":true,"diff_ci95":[-6.207458587592837,8.061834283019177],"p_upper_wins":0.731},{"upper":"system-one-sg","lower":"gpt-6-luna-low","tie":true,"diff_ci95":[-7.198905174854513,2.13399902678581],"p_upper_wins":0.5555},{"upper":"gpt-6-luna-low","lower":"kushal-gemma4-31b-it-autoloops","tie":false,"diff_ci95":[0.790173617458304,1.8994744112441106],"p_upper_wins":1.0},{"upper":"kushal-gemma4-31b-it-autoloops","lower":"gpt-6-luna","tie":true,"diff_ci95":[-0.15165210624559347,0.9136093152573668],"p_upper_wins":0.9195},{"upper":"gpt-6-luna","lower":"decider-2b","tie":true,"diff_ci95":[-8.662717657197522,11.152558684506745],"p_upper_wins":0.5975},{"upper":"decider-2b","lower":"system-one-open","tie":true,"diff_ci95":[-8.289063466492474,10.675191193322593],"p_upper_wins":0.5955},{"upper":"system-one-open","lower":"jevone","tie":true,"diff_ci95":[-6.488940144153428,10.430755862969878],"p_upper_wins":0.627},{"upper":"jevone","lower":"kev-4b","tie":true,"diff_ci95":[-5.578786299067803,12.834833786237505],"p_upper_wins":0.7725},{"upper":"kev-4b","lower":"kev-8b","tie":true,"diff_ci95":[-5.811453110718897,8.711967125065511],"p_upper_wins":0.663},{"upper":"kev-8b","lower":"nimble-9b","tie":true,"diff_ci95":[-6.447837630202695,4.257101970983761],"p_upper_wins":0.521},{"upper":"nimble-9b","lower":"open-alternative-jev","tie":true,"diff_ci95":[-5.207725142199323,9.53438373679304],"p_upper_wins":0.746},{"upper":"open-alternative-jev","lower":"openjev-sglang","tie":true,"diff_ci95":[-6.187693085736844,8.28265429977841],"p_upper_wins":0.5605},{"upper":"openjev-sglang","lower":"decider-35b-a3b","tie":false,"diff_ci95":[0.6782238560410832,2.246661067234712],"p_upper_wins":1.0},{"upper":"decider-35b-a3b","lower":"malkuth-2b","tie":true,"diff_ci95":[-7.279882145048464,9.488440765133443],"p_upper_wins":0.62},{"upper":"malkuth-2b","lower":"open-jev-zefan-9b","tie":true,"diff_ci95":[-6.745064614893428,10.115493583501802],"p_upper_wins":0.5985},{"upper":"open-jev-zefan-9b","lower":"gpt-5.6-luna","tie":false,"diff_ci95":[0.2337847255866267,1.2804668975038942],"p_upper_wins":0.9975},{"upper":"gpt-5.6-luna","lower":"localjev-qwen3.5-4b","tie":true,"diff_ci95":[-5.431958325926747,6.928887144748287],"p_upper_wins":0.672},{"upper":"localjev-qwen3.5-4b","lower":"gemini-3.1-flash-lite","tie":true,"diff_ci95":[-3.5138971722461987,8.733521539966683],"p_upper_wins":0.7415},{"upper":"gemini-3.1-flash-lite","lower":"ninfer-qwen3.8-27b","tie":false,"diff_ci95":[1.1311957685643348,1.8046981717356456],"p_upper_wins":1.0},{"upper":"ninfer-qwen3.8-27b","lower":"decision-2b","tie":true,"diff_ci95":[-5.751623996759211,5.086876526976401],"p_upper_wins":0.516},{"upper":"decision-2b","lower":"openjev-thinking","tie":true,"diff_ci95":[-5.123266312043672,5.864039614968194],"p_upper_wins":0.4925},{"upper":"openjev-thinking","lower":"ninfer-qwen3.8-27b-t1.5","tie":false,"diff_ci95":[0.052523558620332,0.7926896300206859],"p_upper_wins":0.987},{"upper":"ninfer-qwen3.8-27b-t1.5","lower":"instinct","tie":true,"diff_ci95":[-0.23440520735511094,0.27219883350117824],"p_upper_wins":0.5565},{"upper":"instinct","lower":"typecastlm","tie":true,"diff_ci95":[-6.126751728238739,4.832016778084619],"p_upper_wins":0.502},{"upper":"typecastlm","lower":"djev-thinking","tie":true,"diff_ci95":[-4.653677430027907,6.744113169361638],"p_upper_wins":0.552},{"upper":"djev-thinking","lower":"jev-qwen3.5-9b-base-nvfp4","tie":true,"diff_ci95":[-4.457785665220995,5.193343451686781],"p_upper_wins":0.62},{"upper":"jev-qwen3.5-9b-base-nvfp4","lower":"litjev","tie":true,"diff_ci95":[-3.259373791201761,6.0464640970184105],"p_upper_wins":0.661},{"upper":"litjev","lower":"reflex-27b","tie":false,"diff_ci95":[2.7348000637933807,3.200328573928312],"p_upper_wins":1.0},{"upper":"reflex-27b","lower":"raw-phi-4-mini","tie":true,"diff_ci95":[-4.521568447923932,5.072568179434511],"p_upper_wins":0.631},{"upper":"raw-phi-4-mini","lower":"opensourcejev-qwen35-4b-q4km","tie":true,"diff_ci95":[-3.659380215310137,7.717511235132574],"p_upper_wins":0.746},{"upper":"opensourcejev-qwen35-4b-q4km","lower":"open-jev-zefan-2b","tie":true,"diff_ci95":[-0.7450788449351966,6.868849362594874],"p_upper_wins":0.9405},{"upper":"open-jev-zefan-2b","lower":"deepseek-flash","tie":true,"diff_ci95":[-1.3674561632732893,3.525168581204988],"p_upper_wins":0.7915},{"upper":"deepseek-flash","lower":"gliner2-large","tie":true,"diff_ci95":[-3.341068194853924,3.041084700863563],"p_upper_wins":0.5715},{"upper":"gliner2-large","lower":"qwen3-reranker-4b","tie":true,"diff_ci95":[-2.2260314902761027,5.016575343994072],"p_upper_wins":0.779},{"upper":"qwen3-reranker-4b","lower":"simplejev-qwen3.8-27b","tie":true,"diff_ci95":[-0.273819661294457,5.436019165960417],"p_upper_wins":0.9505},{"upper":"simplejev-qwen3.8-27b","lower":"simplejev-qwen3.5-0.8b","tie":true,"diff_ci95":[-1.8829055280393932,2.1008171350669484],"p_upper_wins":0.66},{"upper":"simplejev-qwen3.5-0.8b","lower":"decision-machine-1","tie":true,"diff_ci95":[-1.4217893897136948,3.2162038741644787],"p_upper_wins":0.7435},{"upper":"decision-machine-1","lower":"gliner2.5-multi","tie":true,"diff_ci95":[-1.6973912035421361,2.436169521373138],"p_upper_wins":0.6445},{"upper":"gliner2.5-multi","lower":"gliner2","tie":true,"diff_ci95":[-1.5932857318617524,2.180827374451306],"p_upper_wins":0.63},{"upper":"gliner2","lower":"clm-8b","tie":true,"diff_ci95":[-1.129600446088771,2.8034533953497283],"p_upper_wins":0.7655},{"upper":"clm-8b","lower":"jevact","tie":true,"diff_ci95":[-1.4176142087453059,1.4066417670225329],"p_upper_wins":0.4975},{"upper":"jevact","lower":"kev-0.6b","tie":true,"diff_ci95":[-0.9110435978334234,1.5433406071188291],"p_upper_wins":0.6335},{"upper":"kev-0.6b","lower":"raw-qwen3-0.6b","tie":true,"diff_ci95":[-1.2753644639076047,1.2663424961142864],"p_upper_wins":0.548},{"upper":"raw-qwen3-0.6b","lower":"raw-qwen3-1.7b","tie":true,"diff_ci95":[-0.5569164664604941,0.9749357000561392],"p_upper_wins":0.7},{"upper":"raw-qwen3-1.7b","lower":"gliner2.5-small","tie":true,"diff_ci95":[-1.0679165999301004,1.240171000364177],"p_upper_wins":0.518},{"upper":"gliner2.5-small","lower":"mirror","tie":true,"diff_ci95":[-0.14467571549131245,1.4839676642058017],"p_upper_wins":0.9305},{"upper":"mirror","lower":"zerank-2","tie":true,"diff_ci95":[-0.19884477053933702,0.566713544498838],"p_upper_wins":0.6705},{"upper":"zerank-2","lower":"jeff","tie":true,"diff_ci95":[-0.2748776230456189,0.2529313789425559],"p_upper_wins":0.5635},{"upper":"jeff","lower":"smalljev","tie":true,"diff_ci95":[-0.18024600201515434,0.29883890068258734],"p_upper_wins":0.6425},{"upper":"smalljev","lower":"opendecision","tie":true,"diff_ci95":[-0.08188668460115738,0.24471420587888623],"p_upper_wins":0.7885},{"upper":"opendecision","lower":"bge-reranker-v2-m3","tie":true,"diff_ci95":[0.0,0.13147202005412764],"p_upper_wins":0.892},{"upper":"bge-reranker-v2-m3","lower":"certo","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"certo","lower":"decision-fast","tie":true,"diff_ci95":[-9.99013286476343e-05,0.0],"p_upper_wins":0.0},{"upper":"decision-fast","lower":"gte-reranker-modernbert-base","tie":true,"diff_ci95":[0.0,9.990132864763427e-05],"p_upper_wins":0.0505},{"upper":"gte-reranker-modernbert-base","lower":"kev-0.5b","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"kev-0.5b","lower":"laya","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"laya","lower":"lev-350m","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"lev-350m","lower":"mghafiri-qwen3.5-0.8b-decision-model","tie":true,"diff_ci95":[-0.01495240866961929,0.0],"p_upper_wins":0.0},{"upper":"mghafiri-qwen3.5-0.8b-decision-model","lower":"mxbai-rerank-base-v2","tie":true,"diff_ci95":[0.0,0.014952408669619277],"p_upper_wins":0.475},{"upper":"mxbai-rerank-base-v2","lower":"needle-3","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"needle-3","lower":"needle-3-tools","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"needle-3-tools","lower":"open-jev-deberta-v3-large","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"open-jev-deberta-v3-large","lower":"open-jev-json-canvas-joshuasp","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"open-jev-json-canvas-joshuasp","lower":"openjev-verdict","tie":true,"diff_ci95":[-0.00962844029451886,0.0],"p_upper_wins":0.0},{"upper":"openjev-verdict","lower":"openjev-verdict-1.4","tie":true,"diff_ci95":[0.0,0.0096284402945188],"p_upper_wins":0.325},{"upper":"openjev-verdict-1.4","lower":"qwen3.8-27b","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"qwen3.8-27b","lower":"verdict-small","tie":true,"diff_ci95":[-0.003310895006967587,0.0],"p_upper_wins":0.0},{"upper":"verdict-small","lower":"von-395m","tie":true,"diff_ci95":[0.0,0.003310895006967579],"p_upper_wins":0.1255}]},"C":{"order":["cygnet","winnow-12b","jev-1.13.0","jev-omni","spark-s1-4b-v6","djev","decider-4b-v2","jev-local","malkuth-4b","metask-jev-4b","qwen35-9b-jev-data-mix-v2","raw-qwen3-4b-instruct-2507","semif-qwen3.5-4b","reflex-4b","ninfer-qwen3.8-flash-next","standardone-8b","hopper","swanone","jqv","kushal-gemma4-31b-it-autoloops","gpt-6-luna-low","jevk5-v02","gpt-6-luna","raw-qwen3-8b","nimble-9b","decider-2b","system-one-sg","jevone","system-one-open","openjev-sglang","decider-35b-a3b","kev-4b","open-jev-zefan-9b","kev-8b","open-alternative-jev","gpt-5.6-luna","malkuth-2b","gemini-3.1-flash-lite","ninfer-qwen3.8-27b","ninfer-qwen3.8-27b-t1.5","instinct","openjev-thinking","localjev-qwen3.5-4b","djev-thinking","decision-2b","litjev","typecastlm","jev-qwen3.5-9b-base-nvfp4","reflex-27b","raw-phi-4-mini","opensourcejev-qwen35-4b-q4km","deepseek-flash","open-jev-zefan-2b","gliner2-large","qwen3-reranker-4b","simplejev-qwen3.8-27b","simplejev-qwen3.5-0.8b","decision-machine-1","gliner2.5-multi","gliner2","jevact","clm-8b","kev-0.6b","raw-qwen3-0.6b","gliner2.5-small","raw-qwen3-1.7b","mirror","zerank-2","jeff","smalljev","opendecision","bge-reranker-v2-m3","certo","decision-fast","gte-reranker-modernbert-base","kev-0.5b","laya","lev-350m","mghafiri-qwen3.5-0.8b-decision-model","mxbai-rerank-base-v2","needle-3","needle-3-tools","open-jev-deberta-v3-large","open-jev-json-canvas-joshuasp","openjev-verdict","openjev-verdict-1.4","qwen3.8-27b","verdict-small","von-395m"],"leader_wording":"joint leaders (statistical tie)","markers":[{"upper":"cygnet","lower":"winnow-12b","tie":true,"diff_ci95":[-0.46125459419296033,1.2374474782799036],"p_upper_wins":0.8345},{"upper":"winnow-12b","lower":"jev-1.13.0","tie":false,"diff_ci95":[0.2845795431300456,2.0792169124533744],"p_upper_wins":0.995},{"upper":"jev-1.13.0","lower":"jev-omni","tie":true,"diff_ci95":[-0.4111511855486413,1.4641041378552804],"p_upper_wins":0.837},{"upper":"jev-omni","lower":"spark-s1-4b-v6","tie":false,"diff_ci95":[1.7852740064338062,7.931344922135727],"p_upper_wins":1.0},{"upper":"spark-s1-4b-v6","lower":"djev","tie":true,"diff_ci95":[-0.7448304920344243,5.42524136062795],"p_upper_wins":0.969},{"upper":"djev","lower":"decider-4b-v2","tie":true,"diff_ci95":[-5.3966940918337905,12.827482635373379],"p_upper_wins":0.7765},{"upper":"decider-4b-v2","lower":"jev-local","tie":true,"diff_ci95":[-7.673520278253025,13.685000613835673],"p_upper_wins":0.73},{"upper":"jev-local","lower":"malkuth-4b","tie":true,"diff_ci95":[-7.271315179337868,12.174319028879005],"p_upper_wins":0.694},{"upper":"malkuth-4b","lower":"metask-jev-4b","tie":true,"diff_ci95":[-5.765562613471671,8.507608985220587],"p_upper_wins":0.645},{"upper":"metask-jev-4b","lower":"qwen35-9b-jev-data-mix-v2","tie":true,"diff_ci95":[-5.540261470682385,8.474042015286878],"p_upper_wins":0.639},{"upper":"qwen35-9b-jev-data-mix-v2","lower":"raw-qwen3-4b-instruct-2507","tie":true,"diff_ci95":[-6.9286842199874945,10.012447217823107],"p_upper_wins":0.6435},{"upper":"raw-qwen3-4b-instruct-2507","lower":"semif-qwen3.5-4b","tie":true,"diff_ci95":[-9.439724268362463,9.589503250154001],"p_upper_wins":0.5355},{"upper":"semif-qwen3.5-4b","lower":"reflex-4b","tie":true,"diff_ci95":[-4.192089985162522,8.547530948665214],"p_upper_wins":0.741},{"upper":"reflex-4b","lower":"ninfer-qwen3.8-flash-next","tie":true,"diff_ci95":[-6.340779395292697,8.523949882705208],"p_upper_wins":0.564},{"upper":"ninfer-qwen3.8-flash-next","lower":"standardone-8b","tie":true,"diff_ci95":[-1.2262471900961955,6.302532174322392],"p_upper_wins":0.553},{"upper":"standardone-8b","lower":"hopper","tie":true,"diff_ci95":[-7.501300874741987,6.497417848468792],"p_upper_wins":0.4465},{"upper":"hopper","lower":"swanone","tie":true,"diff_ci95":[-7.003211547109533,8.200550060680202],"p_upper_wins":0.5185},{"upper":"swanone","lower":"jqv","tie":true,"diff_ci95":[-2.58882077686001,10.54440365024724],"p_upper_wins":0.881},{"upper":"jqv","lower":"kushal-gemma4-31b-it-autoloops","tie":true,"diff_ci95":[-4.631558003521666,8.724095922018243],"p_upper_wins":0.671},{"upper":"kushal-gemma4-31b-it-autoloops","lower":"gpt-6-luna-low","tie":true,"diff_ci95":[-0.4144644818337552,0.32506232006125557],"p_upper_wins":0.439},{"upper":"gpt-6-luna-low","lower":"jevk5-v02","tie":true,"diff_ci95":[-7.521983670107476,7.470648883641279],"p_upper_wins":0.5305},{"upper":"jevk5-v02","lower":"gpt-6-luna","tie":true,"diff_ci95":[-5.761754010481389,9.27723766534513],"p_upper_wins":0.6435},{"upper":"gpt-6-luna","lower":"raw-qwen3-8b","tie":true,"diff_ci95":[-0.18601217083374647,13.10257722534698],"p_upper_wins":0.973},{"upper":"raw-qwen3-8b","lower":"nimble-9b","tie":true,"diff_ci95":[-6.017058086414951,6.996061920676411],"p_upper_wins":0.5805},{"upper":"nimble-9b","lower":"decider-2b","tie":true,"diff_ci95":[-5.860093258204187,8.150724279487909],"p_upper_wins":0.621},{"upper":"decider-2b","lower":"system-one-sg","tie":true,"diff_ci95":[-7.49504104204474,6.615573973373475],"p_upper_wins":0.444},{"upper":"system-one-sg","lower":"jevone","tie":true,"diff_ci95":[-5.871274270932872,6.7250623291265],"p_upper_wins":0.5085},{"upper":"jevone","lower":"system-one-open","tie":true,"diff_ci95":[-4.537793515948867,7.619546193636626],"p_upper_wins":0.6765},{"upper":"system-one-open","lower":"openjev-sglang","tie":true,"diff_ci95":[-3.6544717773204343,8.00475244548645],"p_upper_wins":0.7405},{"upper":"openjev-sglang","lower":"decider-35b-a3b","tie":true,"diff_ci95":[-2.7292836938423526,3.3392878330668863],"p_upper_wins":0.654},{"upper":"decider-35b-a3b","lower":"kev-4b","tie":true,"diff_ci95":[-5.098303909355384,7.785814324991241],"p_upper_wins":0.6335},{"upper":"kev-4b","lower":"open-jev-zefan-9b","tie":true,"diff_ci95":[-5.00591212889903,8.123144862184763],"p_upper_wins":0.664},{"upper":"open-jev-zefan-9b","lower":"kev-8b","tie":true,"diff_ci95":[-3.399919251554369,5.4998644396219465],"p_upper_wins":0.6785},{"upper":"kev-8b","lower":"open-alternative-jev","tie":true,"diff_ci95":[-6.07645256840151,5.5380481089055795],"p_upper_wins":0.501},{"upper":"open-alternative-jev","lower":"gpt-5.6-luna","tie":true,"diff_ci95":[-4.457570780521773,6.57872272540185],"p_upper_wins":0.599},{"upper":"gpt-5.6-luna","lower":"malkuth-2b","tie":true,"diff_ci95":[-4.723271895196073,7.7518902560895695],"p_upper_wins":0.696},{"upper":"malkuth-2b","lower":"gemini-3.1-flash-lite","tie":true,"diff_ci95":[-5.018588926275203,7.504508923007906],"p_upper_wins":0.615},{"upper":"gemini-3.1-flash-lite","lower":"ninfer-qwen3.8-27b","tie":false,"diff_ci95":[0.6373929781328598,1.1278886019384695],"p_upper_wins":1.0},{"upper":"ninfer-qwen3.8-27b","lower":"ninfer-qwen3.8-27b-t1.5","tie":false,"diff_ci95":[0.1518437407588788,1.5974680381128072],"p_upper_wins":1.0},{"upper":"ninfer-qwen3.8-27b-t1.5","lower":"instinct","tie":true,"diff_ci95":[-1.043786059846809,0.3114062434715871],"p_upper_wins":0.7675},{"upper":"instinct","lower":"openjev-thinking","tie":true,"diff_ci95":[-0.2902045192155084,0.5988127921420549],"p_upper_wins":0.957},{"upper":"openjev-thinking","lower":"localjev-qwen3.5-4b","tie":true,"diff_ci95":[-4.976315285912141,4.108935218874537],"p_upper_wins":0.51},{"upper":"localjev-qwen3.5-4b","lower":"djev-thinking","tie":true,"diff_ci95":[-3.594898936874596,5.535783254809318],"p_upper_wins":0.588},{"upper":"djev-thinking","lower":"decision-2b","tie":true,"diff_ci95":[-2.5997320186339024,5.799242166885696],"p_upper_wins":0.803},{"upper":"decision-2b","lower":"litjev","tie":true,"diff_ci95":[-3.284084292567798,4.3094539735288135],"p_upper_wins":0.56},{"upper":"litjev","lower":"typecastlm","tie":true,"diff_ci95":[-4.338513656085368,3.871297060471404],"p_upper_wins":0.539},{"upper":"typecastlm","lower":"jev-qwen3.5-9b-base-nvfp4","tie":true,"diff_ci95":[-3.1528768014897937,5.9091577988278345],"p_upper_wins":0.706},{"upper":"jev-qwen3.5-9b-base-nvfp4","lower":"reflex-27b","tie":true,"diff_ci95":[-2.4728097206030606,4.444981395697316],"p_upper_wins":0.6655},{"upper":"reflex-27b","lower":"raw-phi-4-mini","tie":true,"diff_ci95":[-1.2853399369371623,5.969916191132811],"p_upper_wins":0.914},{"upper":"raw-phi-4-mini","lower":"opensourcejev-qwen35-4b-q4km","tie":true,"diff_ci95":[-2.908987435154151,5.661690919465499],"p_upper_wins":0.727},{"upper":"opensourcejev-qwen35-4b-q4km","lower":"deepseek-flash","tie":true,"diff_ci95":[-0.41382649340273375,6.063108082558174],"p_upper_wins":0.9545},{"upper":"deepseek-flash","lower":"open-jev-zefan-2b","tie":true,"diff_ci95":[-1.3658465119638772,2.049709591399384],"p_upper_wins":0.6735},{"upper":"open-jev-zefan-2b","lower":"gliner2-large","tie":true,"diff_ci95":[-2.792120721786518,3.1770012971236907],"p_upper_wins":0.613},{"upper":"gliner2-large","lower":"qwen3-reranker-4b","tie":true,"diff_ci95":[-1.8219008874569327,3.7909270004920264],"p_upper_wins":0.754},{"upper":"qwen3-reranker-4b","lower":"simplejev-qwen3.8-27b","tie":true,"diff_ci95":[-0.38877761491280066,4.091506480440268],"p_upper_wins":0.9235},{"upper":"simplejev-qwen3.8-27b","lower":"simplejev-qwen3.5-0.8b","tie":true,"diff_ci95":[-1.2606831760696502,1.9236208095030196],"p_upper_wins":0.751},{"upper":"simplejev-qwen3.5-0.8b","lower":"decision-machine-1","tie":true,"diff_ci95":[-1.2839445281521746,2.4801358570056755],"p_upper_wins":0.7065},{"upper":"decision-machine-1","lower":"gliner2.5-multi","tie":true,"diff_ci95":[-1.4212805072153578,2.0938508046921256],"p_upper_wins":0.653},{"upper":"gliner2.5-multi","lower":"gliner2","tie":true,"diff_ci95":[-1.287776802002299,1.8975081797160034],"p_upper_wins":0.6605},{"upper":"gliner2","lower":"jevact","tie":true,"diff_ci95":[-0.9565426204632848,2.2390854406798963],"p_upper_wins":0.756},{"upper":"jevact","lower":"clm-8b","tie":true,"diff_ci95":[-1.1822757524487788,1.2503592042920653],"p_upper_wins":0.5205},{"upper":"clm-8b","lower":"kev-0.6b","tie":true,"diff_ci95":[-0.8390300975292277,1.2829866091578235],"p_upper_wins":0.6055},{"upper":"kev-0.6b","lower":"raw-qwen3-0.6b","tie":true,"diff_ci95":[-1.00646587345206,1.1842822281247551],"p_upper_wins":0.5925},{"upper":"raw-qwen3-0.6b","lower":"gliner2.5-small","tie":true,"diff_ci95":[-0.8030059707707977,1.2028769493205096],"p_upper_wins":0.599},{"upper":"gliner2.5-small","lower":"raw-qwen3-1.7b","tie":true,"diff_ci95":[-0.9751023406508271,1.0018250757591387],"p_upper_wins":0.5165},{"upper":"raw-qwen3-1.7b","lower":"mirror","tie":true,"diff_ci95":[-0.2185342151873383,1.4406485089800096],"p_upper_wins":0.892},{"upper":"mirror","lower":"zerank-2","tie":true,"diff_ci95":[-0.1930101759960182,0.5249384876878743],"p_upper_wins":0.666},{"upper":"zerank-2","lower":"jeff","tie":true,"diff_ci95":[-0.26759818787988415,0.24458445166305384],"p_upper_wins":0.561},{"upper":"jeff","lower":"smalljev","tie":true,"diff_ci95":[-0.17596473399737889,0.2908281221333333],"p_upper_wins":0.6425},{"upper":"smalljev","lower":"opendecision","tie":true,"diff_ci95":[-0.08364773409802294,0.24134254730875176],"p_upper_wins":0.7875},{"upper":"opendecision","lower":"bge-reranker-v2-m3","tie":true,"diff_ci95":[0.0,0.13398828883638889],"p_upper_wins":0.892},{"upper":"bge-reranker-v2-m3","lower":"certo","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"certo","lower":"decision-fast","tie":true,"diff_ci95":[-0.00011006907697062774,0.0],"p_upper_wins":0.0},{"upper":"decision-fast","lower":"gte-reranker-modernbert-base","tie":true,"diff_ci95":[0.0,0.00011006907697062771],"p_upper_wins":0.0505},{"upper":"gte-reranker-modernbert-base","lower":"kev-0.5b","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"kev-0.5b","lower":"laya","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"laya","lower":"lev-350m","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"lev-350m","lower":"mghafiri-qwen3.5-0.8b-decision-model","tie":true,"diff_ci95":[-0.015856570469936604,0.0],"p_upper_wins":0.0},{"upper":"mghafiri-qwen3.5-0.8b-decision-model","lower":"mxbai-rerank-base-v2","tie":true,"diff_ci95":[0.0,0.01585657046993659],"p_upper_wins":0.475},{"upper":"mxbai-rerank-base-v2","lower":"needle-3","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"needle-3","lower":"needle-3-tools","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"needle-3-tools","lower":"open-jev-deberta-v3-large","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"open-jev-deberta-v3-large","lower":"open-jev-json-canvas-joshuasp","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"open-jev-json-canvas-joshuasp","lower":"openjev-verdict","tie":true,"diff_ci95":[-0.010250636595762235,0.0],"p_upper_wins":0.0},{"upper":"openjev-verdict","lower":"openjev-verdict-1.4","tie":true,"diff_ci95":[0.0,0.010250636595762173],"p_upper_wins":0.325},{"upper":"openjev-verdict-1.4","lower":"qwen3.8-27b","tie":true,"diff_ci95":[0.0,0.0],"p_upper_wins":0.0},{"upper":"qwen3.8-27b","lower":"verdict-small","tie":true,"diff_ci95":[-0.0035774798530392193,0.0],"p_upper_wins":0.0},{"upper":"verdict-small","lower":"von-395m","tie":true,"diff_ci95":[0.0,0.0035774798530392115],"p_upper_wins":0.1255}]}},"systems":[{"key":"cygnet","display":"Cygnet (blockbrain, frozen Gemma-4-12B-it)","addendum":null,"author":"blockbrain","repo":"https://github.com/blockbrain-ai/cygnet-recipe","class":"system-one-open","licence":"shim MIT; weights Apache-2.0 with Google's Gemma Prohibited Use Policy","open":"yes","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container","gpu":"RTXPRO6000","adapter_id":"images/cygnet/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":71.09241537850588,"calibration":87.00732784869052,"speed":90.9725824502543,"cost":56.42912569481113},"scores":{"A":73.7012674756234,"B":73.16429088621824,"C":73.7012674756234},"views":{"Balanced 33:33:33":70.12644697753653,"Emphasis on Accuracy 60:20:20":70.50966710035375,"Emphasis on Cost 20:20:60":63.92018198047105,"Emphasis on Speed 20:60:20":77.2027746144365,"Intelligence only":71.09241537850588},"composite_ci95":{"A":[72.3623865819622,74.45948779199418],"B":[71.50599842979109,74.36519444510661],"C":[72.3623865819622,74.45948779199418]},"jevbench_score":73.7012674756234,"intelligence":{"I_open":72.46391414608965,"I_sealed":69.7209166109221,"base":71.09241537850588,"gap":2.7429975351675466,"excess":-2.443629964911696,"penalty":1.0,"per_type_split":{"open|choice":{"cc":82.9228973908834,"tiers":{"easy":98.19113656918904,"standard":89.08240187158823,"judge":92.73858921161826,"hard":68.66431649040345},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":57.86443298155948,"tiers":{"easy":76.47058823529412,"standard":69.6969696969697,"judge":72.54901960784315,"hard":36.28318584070796},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":76.6044120658261,"tiers":{"easy":79.71124459421121,"standard":87.82244564147533,"judge":75.83786659638186,"hard":70.79359624798836},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":80.28732165371557,"tiers":{"easy":94.140625,"standard":82.34095539959247,"judge":78.79858657243817,"hard":76.91373025516404},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":55.67272727272728,"tiers":{"easy":84.00000000000001,"standard":52.0,"judge":72.0,"hard":38.18181818181819},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":73.20270090632351,"tiers":{"easy":82.23234484152947,"standard":77.3304059414898,"judge":72.01127703284678,"hard":69.77500531004645},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":87.00732784869052,"parts":{"choice":{"ece_hard":0.04126117192486663,"ece_score":91.74776561502668,"mean_tvd":0.28928215362572635,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":81.40977512622702},"noul":{"ece":0.059567446143809336,"brier":0.09627388769694645,"n":495.0,"score":88.08651077123814,"abstention_rate":0.1393939393939394},"score":{"nrps":0.055629168051636785,"top_ece":0.056928439488117065,"n":330.0,"interval90_coverage":0.9666666666666667,"interval90_width":0.3088383838383839,"score":91.52569764860645}}},"speed":{"p50_s_raw":0.03980663896072656,"p95_s_raw":0.09906632518395769,"p50_s_adjusted":0.2296132779214531,"p95_s_adjusted":0.3481326503679154,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.028337561576354687,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":1,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":1,"B":2,"C":1}},{"key":"winnow-12b","display":"Winnow-12B Q8","addendum":null,"author":"Eldan Ring","repo":"https://huggingface.co/EldanRing/Winnow-12B","class":"jev-rebuild","licence":"Apache-2.0, including the applicable Gemma 4 base/derivative licence terms","open":"yes","underlying":"google/gemma-4-12B-it LoRA fine-tune, merged and exported as Q8_0 GGUF","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX6000), offline read-only container","gpu":"RTX6000","adapter_id":"images/winnow-12b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":74.43468593005443,"calibration":84.06987570289192,"speed":86.13960582383429,"cost":56.556023424877196},"scores":{"A":73.23310265651646,"B":73.47030575448954,"C":73.23310265651646},"views":{"Balanced 33:33:33":70.21610446721299,"Emphasis on Accuracy 60:20:20":71.8448249649829,"Emphasis on Cost 20:20:60":64.02999151081272,"Emphasis on Speed 20:60:20":75.8226400442452,"Intelligence only":74.43468593005443},"composite_ci95":{"A":[72.02170416109966,73.99468092883008],"B":[71.92399627008938,74.62805392246878],"C":[72.02170416109966,73.99468092883008]},"jevbench_score":73.23310265651646,"intelligence":{"I_open":72.47593928648939,"I_sealed":76.39343257361945,"base":74.43468593005443,"gap":-3.917493287130057,"excess":-9.1041207872093,"penalty":1.0,"per_type_split":{"open|choice":{"cc":77.61378737459853,"tiers":{"easy":98.19113656918904,"standard":90.64205874707565,"judge":83.40248962655602,"hard":61.61378770074421},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":58.608048207215305,"tiers":{"easy":82.35294117647058,"standard":75.75757575757575,"judge":66.66666666666667,"hard":38.053097345132734},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":81.20598227765439,"tiers":{"easy":90.95264588576535,"standard":92.03169612810453,"judge":81.13037013155842,"hard":73.41316855997353},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":82.04454751825295,"tiers":{"easy":97.0703125,"standard":83.69934344577767,"judge":82.773851590106,"hard":76.91373025516404},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":66.14545454545454,"tiers":{"easy":100.0,"standard":60.00000000000001,"judge":72.0,"hard":56.363636363636374},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":80.99029565715088,"tiers":{"easy":87.91654862427384,"standard":81.96986364427659,"judge":82.4600539087451,"hard":77.66662973311162},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":84.06987570289192,"parts":{"choice":{"ece_hard":0.08490714326828491,"ece_score":83.01857134634302,"mean_tvd":0.3393316979020659,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":74.54270077806822},"noul":{"ece":0.06312829271927016,"brier":0.08320230663204263,"n":495.0,"score":87.37434145614597,"abstention_rate":0.13131313131313127},"score":{"nrps":0.04821302833936116,"top_ece":0.0729676370857035,"n":330.0,"interval90_coverage":0.9303030303030303,"interval90_width":0.1404040404040404,"score":90.2925848744616}}},"speed":{"p50_s_raw":0.09506642795167863,"p95_s_raw":0.2825698039145205,"p50_s_adjusted":0.3401328559033573,"p95_s_adjusted":0.7151396078290411,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.028062900246305422,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":2,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":2,"B":1,"C":2}},{"key":"jev-1.13.0","display":"Jev 1.13.0 (TypeSafe AI)","addendum":null,"author":"TypeSafe AI","repo":"https://docs.typesafe.ai","class":"jev","licence":"proprietary API","open":"no","underlying":"closed","endpoint_kind":"api","endpoint_condition":"the operator's hosted API","gpu":null,"adapter_id":"harness/specs/jev-1.13.0.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":true,"api_exposure_note":"API measurement: the operator's endpoint received sealed item text, without answers.","status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":71.99974990450261,"calibration":88.02903108992821,"speed":83.8140999985063,"cost":54.72500234484654},"scores":{"A":72.13292897895896,"B":72.10625376369363,"C":72.13292897895896},"views":{"Balanced 33:33:33":68.03756745894525,"Emphasis on Accuracy 60:20:20":69.56893280213254,"Emphasis on Cost 20:20:60":62.00423444459753,"Emphasis on Speed 20:60:20":73.57743121846089,"Intelligence only":71.99974990450261},"composite_ci95":{"A":[71.01427970926099,72.61481293732982],"B":[70.60279184971247,73.05372079639332],"C":[71.01427970926099,72.61481293732982]},"jevbench_score":72.13292897895896,"intelligence":{"I_open":71.54923917772135,"I_sealed":72.45026063128388,"base":71.99974990450261,"gap":-0.9010214535625352,"excess":-6.087648953641778,"penalty":1.0,"per_type_split":{"open|choice":{"cc":85.73455614675619,"tiers":{"easy":98.19113656918904,"standard":98.44034312451261,"judge":92.73858921161826,"hard":71.01449275362319},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":47.75448393355734,"tiers":{"easy":76.47058823529412,"standard":54.54545454545454,"judge":72.54901960784315,"hard":18.584070796460182},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":81.15867745285051,"tiers":{"easy":81.8188800446865,"standard":86.17932862190813,"judge":85.66186107470512,"hard":75.10591350397176},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":87.59399876813663,"tiers":{"easy":97.0703125,"standard":89.13289563051845,"judge":81.4487632508834,"hard":89.0643985419198},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":48.618181818181824,"tiers":{"easy":84.00000000000001,"standard":60.00000000000001,"judge":48.0,"hard":34.54545454545455},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":81.13860130753321,"tiers":{"easy":86.53079710144928,"standard":77.45413836093937,"judge":83.82451303260066,"hard":79.61835003855052},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":88.02903108992821,"parts":{"choice":{"ece_hard":0.027114101301826996,"ece_score":94.5771797396346,"mean_tvd":0.3155481665500016,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":81.51118154231722},"noul":{"ece":0.05597979797979819,"brier":0.07014484848484845,"n":495.0,"score":88.80404040404036,"abstention_rate":0.2282828282828283},"score":{"nrps":0.041089662604001445,"top_ece":0.04173645546372849,"n":330.0,"interval90_coverage":0.9696969696969697,"interval90_width":0.19823232323232326,"score":93.77187132342706}}},"speed":{"p50_s_raw":0.6164783202111721,"p95_s_raw":0.674019081890583,"p50_s_adjusted":0.6164783202111721,"p95_s_adjusted":0.674019081890583,"n":450,"adjustment":"none (hosted API, measured as is)"},"cost":{"kind":"estimate","usd_per_1000":0.032297327586206896,"basis":"operator standard launch list price (interpretation I-1); no exact base-model floor applies"},"rank":3,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":3,"B":3,"C":3}},{"key":"jev-omni","display":"Jev-Omni (akhilaaa3, Gemma-4-12B merged)","addendum":null,"author":"akhilaaa3","repo":"https://huggingface.co/akhilaaa3/Jev-Omni","class":"jev-rebuild","licence":"Apache-2.0, following Gemma 4; dataset rights stated separately by the author","open":"yes","underlying":"google/gemma-4-12B-it fine-tuned and merged, with a trained 256-way decision head","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX6000), offline read-only container","gpu":"RTX6000","adapter_id":"images/jev-omni/spec.json:omni_v15.JevOmni","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":70.49063617669168,"calibration":82.59813256382532,"speed":84.67623138886884,"cost":56.05113155939739},"scores":{"A":71.5005378093621,"B":71.29624911998478,"C":71.5005378093621},"views":{"Balanced 33:33:33":68.43561492071707,"Emphasis on Accuracy 60:20:20":69.24307531155604,"Emphasis on Cost 20:20:60":62.87842598462993,"Emphasis on Speed 20:60:20":74.1221679560491,"Intelligence only":70.49063617669168},"composite_ci95":{"A":[70.21419464201573,72.40139832609583],"B":[69.61520330245101,72.61670152902757],"C":[70.21419464201573,72.40139832609583]},"jevbench_score":71.5005378093621,"intelligence":{"I_open":70.48105345190129,"I_sealed":70.50021890148207,"base":70.49063617669168,"gap":-0.01916544958078248,"excess":-5.205792949660025,"penalty":1.0,"per_type_split":{"open|choice":{"cc":83.36657120240034,"tiers":{"easy":96.38227313837805,"standard":89.08240187158823,"judge":93.7759336099585,"hard":69.44770857814335},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":55.282444433928035,"tiers":{"easy":76.47058823529412,"standard":69.6969696969697,"judge":56.86274509803921,"hard":41.5929203539823},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":72.7941447193755,"tiers":{"easy":79.99649950944763,"standard":86.06273734032244,"judge":73.12472472755394,"hard":64.11132470525015},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":78.70317394647086,"tiers":{"easy":88.28125,"standard":76.9074032148517,"judge":77.47349823321555,"hard":78.12879708383961},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":56.727272727272734,"tiers":{"easy":84.00000000000001,"standard":56.00000000000001,"judge":68.0,"hard":41.81818181818182},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":76.07021003070264,"tiers":{"easy":83.58858680697834,"standard":73.02819652611512,"judge":77.2106969349196,"hard":74.85625741076477},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":82.59813256382532,"parts":{"choice":{"ece_hard":0.08520662096002907,"ece_score":82.95867580799418,"mean_tvd":0.34815184544266786,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":74.0717456318637},"noul":{"ece":0.06459588018096656,"brier":0.08581972089584758,"n":495.0,"score":87.08082396380668,"abstention_rate":0.17575757575757578},"score":{"nrps":0.06883415898624487,"top_ece":0.09916463954882188,"n":330.0,"interval90_coverage":0.9333333333333333,"interval90_width":0.18914141414141417,"score":86.64182809580556}}},"speed":{"p50_s_raw":0.112679542042315,"p95_s_raw":0.37883703656261736,"p50_s_adjusted":0.37535908408463003,"p95_s_adjusted":0.9076740731252347,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.02917173645320197,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":4,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":4,"B":4,"C":4}},{"key":"decider-4b-v2","display":"decider-4b v2 (Mapika)","addendum":null,"author":"Mapika","repo":"https://github.com/Mapika/decider","class":"system-one-open","licence":"Apache-2.0 (package and weights)","open":"yes","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX5090), offline read-only container","gpu":"RTX5090","adapter_id":"images/decider-4b-v2/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":55.7709920864861,"calibration":85.59400811608691,"speed":90.85503597842234,"cost":64.54376873042901},"scores":{"A":71.28417523518583,"B":67.5275035635358,"C":61.58959786188317},"views":{"Balanced 33:33:33":67.52137451850446,"Emphasis on Accuracy 60:20:20":62.273238587557444,"Emphasis on Cost 20:20:60":66.29796164251424,"Emphasis on Speed 20:60:20":75.25194627944218,"Intelligence only":55.7709920864861},"composite_ci95":{"A":[69.11045953172504,72.34691401220934],"B":[64.68875564545965,69.20814174207187],"C":[50.887685275957445,69.72615382450459]},"jevbench_score":71.28417523518583,"intelligence":{"I_open":61.74364566303797,"I_sealed":49.79833850993424,"base":55.7709920864861,"gap":11.945307153103727,"excess":6.758679653024484,"penalty":1.0,"per_type_split":{"open|choice":{"cc":80.43472694605762,"tiers":{"easy":98.19113656918904,"standard":92.20171562256304,"judge":82.36514522821577,"hard":68.66431649040345},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":42.918461028820204,"tiers":{"easy":76.47058823529412,"standard":54.54545454545454,"judge":70.58823529411764,"hard":7.964601769911495},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":61.87774901423609,"tiers":{"easy":47.07476958525345,"standard":70.18886925795053,"judge":58.8005897771953,"hard":63.730803177405114},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":77.31230284100064,"tiers":{"easy":85.3515625,"standard":78.2657912610369,"judge":76.14840989399293,"hard":75.69866342648847},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":18.727272727272723,"tiers":{"easy":60.00000000000001,"standard":24.0,"judge":24.0,"hard":1.8181818181818077},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":53.35543996152937,"tiers":{"easy":66.72291666666668,"standard":61.24424853064653,"judge":52.99970513900589,"hard":46.335967617579065},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":85.59400811608691,"parts":{"choice":{"ece_hard":0.04920685920577605,"ece_score":90.15862815884479,"mean_tvd":0.22729216655000173,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":83.71470575192231},"noul":{"ece":0.05604383838383825,"brier":0.10456726022222225,"n":495.0,"score":88.79123232323235,"abstention_rate":0.26868686868686864},"score":{"nrps":0.09234675938636361,"top_ece":0.11106575757575754,"n":330.0,"interval90_coverage":0.9757575757575757,"interval90_width":0.5386363636363639,"score":84.27608627310607}}},"speed":{"p50_s_raw":0.026405358512420207,"p95_s_raw":0.12747691074036993,"p50_s_adjusted":0.2028107170248404,"p95_s_adjusted":0.4049538214807399,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.015201077586206898,"basis":"documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)"},"rank":5,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":5,"B":5,"C":7}},{"key":"semif-qwen3.5-4b","display":"SemIf, formerly OpenJev (Qwen3.5-4B, TheoLeeCJ)","addendum":null,"author":"Theodore Lee (TheoLeeCJ)","repo":"https://github.com/TheoLeeCJ/openjev","class":"jev-rebuild","licence":"MIT (code); Qwen3.5 weights Apache-2.0","open":"yes","underlying":"Qwen/Qwen3.5-4B (frozen, BF16)","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX5090), offline read-only container","gpu":"RTX5090","adapter_id":"images/semif-qwen3.5-4b/spec.json:remote_inproc_v15.RemoteInprocV15Adapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":51.31227416736609,"calibration":83.9694498943646,"speed":90.890351971217,"cost":63.07359395281681},"scores":{"A":68.66437558511035,"B":64.31456686948948,"C":50.21939777970113},"views":{"Balanced 33:33:33":64.73151414788424,"Emphasis on Accuracy 60:20:20":58.60132233439368,"Emphasis on Cost 20:20:60":64.0579958758946,"Emphasis on Speed 20:60:20":73.15308917543874,"Intelligence only":51.31227416736609},"composite_ci95":{"A":[60.05966147761982,70.15877841526802],"B":[55.6917195697546,66.46721380623629],"C":[41.7080982483471,58.662504296040126]},"jevbench_score":68.66437558511035,"intelligence":{"I_open":53.996219382373376,"I_sealed":48.62832895235881,"base":51.31227416736609,"gap":5.367890430014569,"excess":0.18126292993532633,"penalty":1.0,"per_type_split":{"open|choice":{"cc":73.79929570880981,"tiers":{"easy":94.57340970756708,"standard":82.84377436963868,"judge":89.6265560165975,"hard":52.21308264786526},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":30.297824680958463,"tiers":{"easy":58.823529411764696,"standard":51.515151515151516,"judge":52.941176470588225,"hard":-4.424778761061942},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":57.891537757351855,"tiers":{"easy":38.14197517572985,"standard":75.65692943620333,"judge":57.516282294121936,"hard":54.227674160754056},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":70.95783711255453,"tiers":{"easy":79.4921875,"standard":72.83223907629615,"judge":58.92226148409894,"hard":76.91373025516404},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":13.927272727272719,"tiers":{"easy":52.0,"standard":15.999999999999993,"judge":15.999999999999993,"hard":1.8181818181818077},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":60.99987701724918,"tiers":{"easy":76.14710035023134,"standard":63.67894900857971,"judge":60.24526285169589,"hard":56.43949581250334},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":83.9694498943646,"parts":{"choice":{"ece_hard":0.06314318276148391,"ece_score":87.37136344770322,"mean_tvd":0.2735131150426244,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":80.01002597172038},"noul":{"ece":0.06720214043427124,"brier":0.1307654521663549,"n":495.0,"score":86.55957191314575,"abstention_rate":0.2909090909090909},"score":{"nrps":0.10609041896052611,"top_ece":0.09356727253746004,"n":330.0,"interval90_coverage":0.9454545454545454,"interval90_width":0.5229797979797979,"score":85.33875179822769}}},"speed":{"p50_s_raw":0.039298868738114834,"p95_s_raw":0.10318161030299944,"p50_s_adjusted":0.22859773747622966,"p95_s_adjusted":0.3563632206059989,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.01701689039408867,"basis":"documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)"},"rank":6,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":6,"B":8,"C":13}},{"key":"spark-s1-4b-v6","display":"spark-s1-4b-v6 (Open Spark Jev, abhishek085)","addendum":null,"author":"Abhishek Rai (abhishek085)","repo":"https://github.com/abhishek085/open-spark-jev","class":"jev-rebuild","licence":"Apache-2.0 (code and weights); base Qwen/Qwen3.5-4B Apache-2.0","open":"yes","underlying":"Qwen/Qwen3.5-4B with a LoRA, read at the option-letter logits with a fitted temperature (no extra head)","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX6000), offline read-only container","gpu":"RTX6000","adapter_id":"images/spark-s1-4b-v6/spec.json:spark_v15.Spark","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":62.11757005795365,"calibration":69.70580960824081,"speed":85.52672016213519,"cost":60.422063196854964},"scores":{"A":68.15818403751406,"B":66.85786733898678,"C":68.15818403751406},"views":{"Balanced 33:33:33":67.65746810827054,"Emphasis on Accuracy 60:20:20":65.32701629026283,"Emphasis on Cost 20:20:60":64.56486948416433,"Emphasis on Speed 20:60:20":73.82742716682645,"Intelligence only":62.11757005795365},"composite_ci95":{"A":[66.27191706094285,69.70470030753586],"B":[64.54463023423625,68.79399704219207],"C":[62.94461884952748,69.70470030753586]},"jevbench_score":68.15818403751406,"intelligence":{"I_open":62.460678071910536,"I_sealed":61.77446204399676,"base":62.11757005795365,"gap":0.6862160279137726,"excess":-4.50041147216547,"penalty":1.0,"per_type_split":{"open|choice":{"cc":72.55232891520762,"tiers":{"easy":94.57340970756708,"standard":82.84377436963868,"judge":86.51452282157676,"hard":51.42969056012534},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":50.43601028504724,"tiers":{"easy":64.70588235294117,"standard":69.6969696969697,"judge":70.58823529411764,"hard":22.123893805309724},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":64.39369501547677,"tiers":{"easy":87.17913149779508,"standard":74.65259596167797,"judge":61.1941893149865,"hard":55.967514697164255},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":73.1148440968712,"tiers":{"easy":79.4921875,"standard":71.47385103011095,"judge":73.49823321554771,"hard":72.05346294046174},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":46.472727272727276,"tiers":{"easy":76.0,"standard":28.000000000000004,"judge":60.00000000000001,"hard":38.18181818181819},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":65.73581476239181,"tiers":{"easy":68.26278642742467,"standard":64.68169284072869,"judge":62.18712724711797,"hard":68.29264844342052},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":69.70580960824081,"parts":{"choice":{"ece_hard":0.16689755974842224,"ece_score":66.62048805031556,"mean_tvd":0.35435283393590955,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":65.5926023283623},"noul":{"ece":0.1283910387499207,"brier":0.1442645809272019,"n":495.0,"score":74.32179225001586,"abstention_rate":0.1333333333333333},"score":{"nrps":0.11390986143211532,"top_ece":0.2510147268204994,"n":330.0,"interval90_coverage":0.7515151515151515,"interval90_width":0.08106060606060606,"score":69.20303424634429}}},"speed":{"p50_s_raw":0.1279726339271292,"p95_s_raw":0.2700090640457343,"p50_s_adjusted":0.40594526785425844,"p95_s_adjusted":0.6900181280914687,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.020857610837438423,"basis":"documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)"},"rank":7,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":7,"B":6,"C":5}},{"key":"metask-jev-4b","display":"metask-jev-4b","addendum":null,"author":"Wayfind (metask-ai)","repo":"https://github.com/metask-ai/metask-jev","class":"jev-rebuild","licence":"Apache-2.0","open":"yes","underlying":"Qwen3.5-4B merged r16 LoRA, candidate-logit readout","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX5090), offline read-only container","gpu":"RTX5090","adapter_id":"images/metask-jev-4b/spec.json:metask_v15.MetaskJevWarm","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1622},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":53.478671436751014,"calibration":82.74360182521858,"speed":89.49053928541517,"cost":57.7311327584882},"scores":{"A":67.47568667137665,"B":64.11928820546953,"C":53.6050902247502},"views":{"Balanced 33:33:33":63.565940709101675,"Emphasis on Accuracy 60:20:20":59.10642509551968,"Emphasis on Cost 20:20:60":61.095985088875466,"Emphasis on Speed 20:60:20":71.89711825951035,"Intelligence only":53.478671436751014},"composite_ci95":{"A":[65.39080983744157,68.69256296806242],"B":[61.58172699270945,65.90468584974295],"C":[45.52555578171516,61.50937969724669]},"jevbench_score":67.47568667137665,"intelligence":{"I_open":56.74073420693068,"I_sealed":50.216608666571354,"base":53.478671436751014,"gap":6.524125540359329,"excess":1.337498040280086,"penalty":1.0,"per_type_split":{"open|choice":{"cc":70.10893404347104,"tiers":{"easy":98.19113656918904,"standard":78.1648037431765,"judge":81.32780082987551,"hard":50.64629847238542},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":43.229220891896574,"tiers":{"easy":76.47058823529412,"standard":66.66666666666667,"judge":58.823529411764696,"hard":11.504424778761058},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":56.88404768542444,"tiers":{"easy":32.011664274608734,"standard":69.38352511074332,"judge":53.48129432018609,"hard":59.4044698493977},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":72.41221362515527,"tiers":{"easy":82.421875,"standard":75.5490151686665,"judge":64.22261484098941,"hard":74.48359659781289},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":25.2,"tiers":{"easy":52.0,"standard":24.0,"judge":24.0,"hard":19.999999999999996},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":53.03761237455879,"tiers":{"easy":67.18940475884378,"standard":57.50830908544155,"judge":54.88502353109698,"hard":45.87875755564252},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0025031289111389077},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":82.74360182521858,"parts":{"choice":{"ece_hard":0.04753605370252558,"ece_score":90.49278925949488,"mean_tvd":0.2525412810871565,"n_tvd":50.0,"n":797.0,"n_ece_hard":275.0,"score":82.61933057538961},"noul":{"ece":0.07502700184001239,"brier":0.13271058288209991,"n":495.0,"score":84.99459963199753,"abstention_rate":0.21212121212121215},"score":{"nrps":0.11350190609753398,"top_ece":0.1370802942685472,"n":330.0,"interval90_coverage":0.9606060606060606,"interval90_width":0.5760101010101012,"score":80.61687526826859}}},"speed":{"p50_s_raw":0.06768286094302312,"p95_s_raw":0.12202179493033323,"p50_s_adjusted":0.28536572188604625,"p95_s_adjusted":0.39404358986066645,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.025642610837438424,"basis":"documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)"},"rank":8,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":8,"B":9,"C":10}},{"key":"hopper","display":"Hopper","addendum":null,"author":"HopitAI","repo":"https://huggingface.co/HopitAI/hopper","class":"jev-rebuild","licence":"Component-specific terms recorded in RESULT.md; submitted adapter release and Qwen base retain their respective terms","open":"yes","underlying":"Qwen3.5-4B plus HopitAI/hopper LoRA","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/hopper/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":49.86320336082703,"calibration":87.86903672712391,"speed":87.23386153386807,"cost":62.25627699896333},"scores":{"A":67.47015061443733,"B":62.932238759063466,"C":46.85427126002592},"views":{"Balanced 33:33:33":62.70591910312661,"Emphasis on Accuracy 60:20:20":56.70702953180434,"Emphasis on Cost 20:20:60":62.38757962057503,"Emphasis on Speed 20:60:20":70.52661677934104,"Intelligence only":49.590731857742696},"composite_ci95":{"A":[56.33592438734226,69.15394144849903],"B":[51.98455577329755,65.27281473616978],"C":[39.12216971343212,54.929785035322034]},"jevbench_score":67.47015061443733,"intelligence":{"I_open":48.7045625282528,"I_sealed":51.02184419340125,"base":49.86320336082703,"gap":-2.3172816651484496,"excess":-7.503909165227692,"penalty":1.0,"per_type_split":{"open|choice":{"cc":70.1240393038206,"tiers":{"easy":98.19113656918904,"standard":82.84377436963868,"judge":71.99170124481329,"hard":55.34665099882491},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":10.233937500985915,"tiers":{"easy":70.58823529411764,"standard":42.42424242424243,"judge":0.0,"hard":-13.27433628318584},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":65.7557107799519,"tiers":{"easy":48.33854622640297,"standard":78.65058411480577,"judge":69.38652302338251,"hard":60.93945606833921},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":75.48169096739608,"tiers":{"easy":85.3515625,"standard":74.19062712248135,"judge":69.52296819787986,"hard":78.12879708383961},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":12.763636363636362,"tiers":{"easy":60.00000000000001,"standard":15.999999999999993,"judge":24.0,"hard":-9.090909090909093},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":64.82020524917131,"tiers":{"easy":79.23209224282715,"standard":67.59502487034688,"judge":65.64745705143598,"hard":59.20938483847106},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":87.86903672712391,"parts":{"choice":{"ece_hard":0.047132288889173826,"ece_score":90.57354222216524,"mean_tvd":0.15466886536122398,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":87.55332784302142},"noul":{"ece":0.07041272261623366,"brier":0.10425515982817155,"n":495.0,"score":85.91745547675328,"abstention_rate":0.406060606060606},"score":{"nrps":0.0863650042965921,"top_ece":0.05545422923573362,"n":330.0,"interval90_coverage":0.9515151515151515,"interval90_width":0.41489898989898993,"score":90.13632686159704}}},"speed":{"p50_s_raw":0.12245992664247751,"p95_s_raw":0.16437284634448568,"p50_s_adjusted":0.39491985328495505,"p95_s_adjusted":0.4787456926889714,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.018118577586206896,"basis":"documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)"},"rank":9,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":9,"B":12,"C":17}},{"key":"malkuth-4b","display":"Malkuth-4B (newfull5, Kev post-train)","addendum":null,"author":"newfull5 (dhtocks)","repo":"https://github.com/newfull5/malkuth","class":"jev-rebuild","licence":"CC-BY-NC-4.0, research use only (XNLI and RACE in the training mix)","open":"yes","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX5090), offline read-only container","gpu":"RTX5090","adapter_id":"images/malkuth-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":54.45108712587745,"calibration":83.20783168346826,"speed":86.15537413790446,"cost":55.82433433950372},"scores":{"A":66.77408647378738,"B":63.88259181735141,"C":54.994412178529096},"views":{"Balanced 33:33:33":62.64960437870624,"Emphasis on Accuracy 60:20:20":59.090765182620935,"Emphasis on Cost 20:20:60":59.72855893426261,"Emphasis on Speed 20:60:20":70.32422767798526,"Intelligence only":54.45108712587746},"composite_ci95":{"A":[64.936284491473,67.89009495158388],"B":[61.571100423606644,65.54784266334441],"C":[46.8770023830624,62.78108641487751]},"jevbench_score":66.77408647378738,"intelligence":{"I_open":58.362772871720175,"I_sealed":50.53940138003472,"base":54.45108712587745,"gap":7.823371491685457,"excess":2.636743991606214,"penalty":1.0,"per_type_split":{"open|choice":{"cc":71.16861524959509,"tiers":{"easy":96.38227313837805,"standard":87.52274499610087,"judge":83.40248962655602,"hard":47.51273012142576},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":39.480542015679966,"tiers":{"easy":76.47058823529412,"standard":51.515151515151516,"judge":70.58823529411764,"hard":0.8849557522123908},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":64.43916134988548,"tiers":{"easy":36.32776497695851,"standard":78.3743816254417,"judge":66.15386631716908,"hard":63.21337157987643},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":69.55316310303347,"tiers":{"easy":76.56249999999999,"standard":71.47385103011095,"judge":64.22261484098941,"hard":70.83839611178615},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":22.836363636363632,"tiers":{"easy":76.0,"standard":15.999999999999993,"judge":28.000000000000004,"hard":9.090909090909083},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":59.228677400707056,"tiers":{"easy":77.18079710144929,"standard":67.53711167086482,"judge":51.13374052232519,"hard":56.657632999229},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":83.20783168346826,"parts":{"choice":{"ece_hard":0.043919494584837536,"ece_score":91.2161010830325,"mean_tvd":0.2473586666000066,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":83.24011721151592},"noul":{"ece":0.10944020202020216,"brier":0.14711016430303037,"n":495.0,"score":78.11195959595956,"abstention_rate":0.2282828282828283},"score":{"nrps":0.09267163514141413,"top_ece":0.07094999999999994,"n":330.0,"interval90_coverage":0.9606060606060606,"interval90_width":0.43181818181818166,"score":88.2714182429293}}},"speed":{"p50_s_raw":0.14595738402567804,"p95_s_raw":0.19921681064297442,"p50_s_adjusted":0.4419147680513561,"p95_s_adjusted":0.5484336212859489,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.02968398399014778,"basis":"price floor: base-model reference price applied; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)"},"rank":10,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":10,"B":10,"C":9}},{"key":"reflex-4b","display":"reflex 4B (kshetrajna12)","addendum":null,"author":"kshetrajna12","repo":"https://github.com/kshetrajna12/reflex","class":"jev-rebuild","licence":"MIT (code, adapter); Apache-2.0 (base)","open":"yes","underlying":"Qwen/Qwen3.5-4B + kshetrajna12/reflex-qwen3.5-4b-lora","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/reflex-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":51.49107492301285,"calibration":86.84320054497626,"speed":68.77483167692569,"cost":63.142712976263866},"scores":{"A":65.23881652418177,"B":61.93175130880487,"C":48.04713427572403},"views":{"Balanced 33:33:33":60.24316014490931,"Emphasis on Accuracy 60:20:20":56.408026714558744,"Emphasis on Cost 20:20:60":61.370427494798925,"Emphasis on Speed 20:60:20":63.38855699417101,"Intelligence only":51.49107492301285},"composite_ci95":{"A":[58.56011597033304,66.34964401610556],"B":[55.04915488817,63.722007658620804],"C":[40.66674720162017,55.96422378625981]},"jevbench_score":65.23881652418177,"intelligence":{"I_open":55.20563662766551,"I_sealed":47.77651321836019,"base":51.49107492301285,"gap":7.429123409305319,"excess":2.2424959092260766,"penalty":1.0,"per_type_split":{"open|choice":{"cc":73.66464105000873,"tiers":{"easy":96.38227313837805,"standard":78.1648037431765,"judge":92.73858921161826,"hard":51.42969056012534},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":33.33349107945673,"tiers":{"easy":70.58823529411764,"standard":45.45454545454546,"judge":49.01960784313726,"hard":6.194690265486735},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":58.618777753531084,"tiers":{"easy":30.865999999999982,"standard":72.12921687279152,"judge":57.9468256880734,"hard":59.30571668137689},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":69.3128758568003,"tiers":{"easy":82.421875,"standard":74.19062712248135,"judge":62.897526501766784,"hard":68.408262454435},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":19.454545454545457,"tiers":{"easy":52.0,"standard":12.00000000000001,"judge":8.000000000000007,"hard":23.636363636363633},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":54.56211834373482,"tiers":{"easy":70.11713768115942,"standard":60.228924013434096,"judge":50.31963774220725,"hard":51.021821125674684},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":86.84320054497626,"parts":{"choice":{"ece_hard":0.06095504332129953,"ece_score":87.80899133574009,"mean_tvd":0.24011401655000167,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":81.89879484036996},"noul":{"ece":0.03688168282828298,"brier":0.1222939817592503,"n":495.0,"score":92.6236634343434,"abstention_rate":0.2909090909090909},"score":{"nrps":0.10425772067447935,"top_ece":0.08779970606060608,"n":330.0,"interval90_coverage":0.9666666666666667,"interval90_width":0.6209595959595962,"score":86.00714336021542}}},"speed":{"p50_s_raw":1.3576662300620228,"p95_s_raw":2.238725413940847,"p50_s_adjusted":2.8653324601240455,"p95_s_adjusted":4.627450827881694,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.016926853448275863,"basis":"documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)"},"rank":11,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":11,"B":13,"C":14}},{"key":"jev-local","display":"jev-local (Qwen3.5-9B)","addendum":null,"author":"us (GitHub)","repo":"https://github.com/us/jev-local","class":"jev-rebuild","licence":"no licence stated in the repository (public code); Apache-2.0 base weights","open":"yes","underlying":"Qwen/Qwen3.5-9B, frozen, per-option mean log-probability","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/jev-local/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":56.27600615219267,"calibration":77.82254243294076,"speed":72.97538812549428,"cost":58.85554343293177},"scores":{"A":65.23868668964282,"B":63.224813137015204,"C":57.391720704958274},"views":{"Balanced 33:33:33":61.902172672191675,"Emphasis on Accuracy 60:20:20":59.52190326603423,"Emphasis on Cost 20:20:60":60.64643903259494,"Emphasis on Speed 20:60:20":65.90214393582374,"Intelligence only":56.27600615219267},"composite_ci95":{"A":[63.46433946165809,66.49644591562449],"B":[61.014275918200745,65.01397981407818],"C":[49.030955191671445,65.92078049729584]},"jevbench_score":65.23868668964282,"intelligence":{"I_open":58.180064462983424,"I_sealed":54.37194784140192,"base":56.27600615219267,"gap":3.8081166215815045,"excess":-1.378510878497738,"penalty":1.0,"per_type_split":{"open|choice":{"cc":66.88423583037286,"tiers":{"easy":90.95568284594513,"standard":71.92617624122693,"judge":81.32780082987551,"hard":47.51273012142576},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":46.944615336078115,"tiers":{"easy":94.11764705882352,"standard":57.57575757575757,"judge":66.66666666666667,"hard":15.04424778761062},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":60.7113422224993,"tiers":{"easy":34.579953917050645,"standard":76.4059628975265,"judge":59.867169069462655,"hard":60.030008826125325},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":65.39152308484063,"tiers":{"easy":61.91406249999999,"standard":61.96513470681459,"judge":61.572438162544174,"hard":70.83839611178615},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":37.01818181818182,"tiers":{"easy":52.0,"standard":12.00000000000001,"judge":52.0,"hard":34.54545454545455},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":60.7061386211833,"tiers":{"easy":77.73496376811593,"standard":72.42779177162049,"judge":53.71099410278012,"hard":55.83446414803394},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":77.82254243294076,"parts":{"choice":{"ece_hard":0.10431407942238266,"ece_score":79.13718411552347,"mean_tvd":0.2741831665500017,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":75.85943373026166},"noul":{"ece":0.131251515151515,"brier":0.15351713593939395,"n":495.0,"score":73.749696969697,"abstention_rate":0.1515151515151515},"score":{"nrps":0.10504582559848477,"top_ece":0.10889212121212123,"n":330.0,"interval90_coverage":0.8757575757575757,"interval90_width":0.26313131313131316,"score":83.85849659886364}}},"speed":{"p50_s_raw":0.3970644664950669,"p95_s_raw":2.594315356318838,"p50_s_adjusted":0.9441289329901338,"p95_s_adjusted":5.338630712637676,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.023522413793103446,"basis":"price floor: base-model reference price applied"},"rank":12,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":12,"B":11,"C":8}},{"key":"djev","display":"djev (Maisa, diffusion-gemma)","addendum":null,"author":"Maisa (David Villalón)","repo":"https://github.com/Davipar/djev-dev","class":"jev-rebuild","licence":"Apache-2.0 code; Google DiffusionGemma Apache-2.0 weights; no djev-specific weights","open":"yes","underlying":"inference method on google/diffusiongemma-26B-A4B-it (one structured denoising read), not a separately trained model","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/djev/spec.json:jevbench.adapters.djev.DjevAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":72.32562987104993,"calibration":80.4134255723618,"speed":91.0029245322729,"cost":48.22031380545122},"scores":{"A":64.15569234453473,"B":64.75500708450079,"C":64.15569234453473},"views":{"Balanced 33:33:33":61.25235241364572,"Emphasis on Accuracy 60:20:20":63.52491736662031,"Emphasis on Cost 20:20:60":53.43474942542091,"Emphasis on Speed 20:60:20":68.86368074385312,"Intelligence only":67.26858316507415},"composite_ci95":{"A":[63.06972830768684,65.00414152390454],"B":[63.369610740199924,65.92487018708866],"C":[63.06972830768684,65.00414152390454]},"jevbench_score":64.15569234453473,"intelligence":{"I_open":72.70481821582241,"I_sealed":71.94644152627744,"base":72.32562987104993,"gap":0.7583766895449742,"excess":-4.428250810534268,"penalty":1.0,"per_type_split":{"open|choice":{"cc":76.61741501146423,"tiers":{"easy":94.57340970756708,"standard":92.20171562256304,"judge":86.51452282157676,"hard":56.91343517430473},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":66.16361427917909,"tiers":{"easy":82.35294117647058,"standard":78.78787878787878,"judge":80.3921568627451,"hard":45.13274336283186},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":75.33342535682394,"tiers":{"easy":85.959739463197,"standard":85.35101065815556,"judge":72.92196305695782,"hard":69.47665090446445},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":72.7798797008079,"tiers":{"easy":85.3515625,"standard":72.83223907629615,"judge":69.52296819787986,"hard":72.05346294046174},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":70.25454545454545,"tiers":{"easy":92.0,"standard":63.99999999999999,"judge":76.0,"hard":63.63636363636365},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":72.80489942347899,"tiers":{"easy":76.59060844866048,"standard":71.952289347639,"judge":70.41666194673873,"hard":74.07595531265878},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":80.4134255723618,"parts":{"choice":{"ece_hard":0.12015275137219117,"ece_score":75.96944972556176,"mean_tvd":0.3136562430742985,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":72.30191270906596},"noul":{"ece":0.0897244771539998,"brier":0.0956660081741808,"n":495.0,"score":82.05510456920004,"abstention_rate":0.07070707070707072},"score":{"nrps":0.0693604150006043,"top_ece":0.09648719811150377,"n":330.0,"interval90_coverage":0.9,"interval90_width":0.1881313131313132,"score":86.88325943881941}}},"speed":{"p50_s_raw":0.050276658963412046,"p95_s_raw":0.08340810290537767,"p50_s_adjusted":0.2505533179268241,"p95_s_adjusted":0.3168162058107553,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.05320954433497537,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":13,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":13,"B":7,"C":6}},{"key":"raw-qwen3-4b-instruct-2507","display":"Raw Qwen3 4B Instruct 2507 direct logits","addendum":null,"author":"Alibaba Qwen / neutral reproduction","repo":"https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507","class":"raw-logit-control","licence":"Apache-2.0","open":"yes","underlying":"Qwen3-4B-Instruct-2507 BF16","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/raw-qwen3-4b-instruct-2507/spec.json:raw_adapter.RawControlAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":54.06280185198723,"calibration":52.94631488507357,"speed":89.228241580316,"cost":63.36068112101963},"scores":{"A":62.13695038108371,"B":60.334779733788544,"C":50.44806735123544},"views":{"Balanced 33:33:33":65.95308478978349,"Emphasis on Accuracy 60:20:20":60.62010074266458,"Emphasis on Cost 20:20:60":64.8910768522085,"Emphasis on Speed 20:60:20":73.63628483823207,"Intelligence only":54.06280185198723},"composite_ci95":{"A":[59.35165730733617,64.21780469584392],"B":[57.342469645630096,62.736547541702414],"C":[41.992589504404044,59.11349905782746]},"jevbench_score":62.13695038108371,"intelligence":{"I_open":57.42840197206909,"I_sealed":50.69720173190538,"base":54.06280185198723,"gap":6.731200240163709,"excess":1.5445727400844662,"penalty":1.0,"per_type_split":{"open|choice":{"cc":65.49764059357105,"tiers":{"easy":92.7645462767561,"standard":75.04548999220171,"judge":80.29045643153528,"hard":42.81237759498628},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":49.80707649109523,"tiers":{"easy":82.35294117647058,"standard":42.42424242424243,"judge":78.43137254901961,"hard":23.893805309734507},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":56.980488831541,"tiers":{"easy":37.16373879798242,"standard":74.5345292714069,"judge":52.065630628239326,"hard":56.84379977247396},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":63.45679070007781,"tiers":{"easy":76.56249999999999,"standard":71.47385103011095,"judge":53.621908127208485,"hard":63.54799513973269},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":38.036363636363646,"tiers":{"easy":12.00000000000001,"standard":32.00000000000001,"judge":36.00000000000001,"hard":49.09090909090909},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":50.598450859274706,"tiers":{"easy":71.82574190481856,"standard":64.809117602594,"judge":45.36051487860803,"hard":42.11474671172911},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":52.94631488507357,"parts":{"choice":{"ece_hard":0.32878554147980255,"ece_score":34.242891704039494,"mean_tvd":0.40842296047137,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":46.70029782845124},"noul":{"ece":0.22707429711655797,"brier":0.23134890441078815,"n":495.0,"score":54.58514057668841,"abstention_rate":0.0444444444444444},"score":{"nrps":0.15100753264429193,"top_ece":0.3489611711770434,"n":330.0,"interval90_coverage":0.6848484848484848,"interval90_width":0.11994949494949493,"score":57.55350625008107}}},"speed":{"p50_s_raw":0.05585077637806535,"p95_s_raw":0.153212562156841,"p50_s_adjusted":0.2617015527561307,"p95_s_adjusted":0.456425124313682,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.01664602832512315,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":14,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":14,"B":14,"C":12}},{"key":"jqv","display":"jqv (Qwen3-32B zero-shot)","addendum":null,"author":"hjmurmur (Octalab)","repo":"https://github.com/Octalab-Inc/jqv","class":"jev-rebuild","licence":"Apache-2.0 (Qwen3-32B weights); serving code public","open":"yes","underlying":"Qwen/Qwen3-32B, bf16, read as a direct-logit classifier (no fine-tuning)","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/jqv/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":49.086814605278015,"calibration":86.76753584348495,"speed":83.36030040456406,"cost":51.172592607797476},"scores":{"A":60.77445997498618,"B":57.50151945806778,"C":42.2044860937404},"views":{"Balanced 33:33:33":55.70071491439261,"Emphasis on Accuracy 60:20:20":52.01101730934296,"Emphasis on Cost 20:20:60":52.96026958495173,"Emphasis on Speed 20:60:20":63.49005579723604,"Intelligence only":47.310173664858446},"composite_ci95":{"A":[51.25244150522343,64.26095073970949],"B":[48.000035915340774,61.47780194813299],"C":[35.591973267516266,49.209015623794464]},"jevbench_score":60.77445997498618,"intelligence":{"I_open":51.3394552913734,"I_sealed":46.83417391918263,"base":49.086814605278015,"gap":4.505281372190765,"excess":-0.6813461278884771,"penalty":1.0,"per_type_split":{"open|choice":{"cc":72.99961235350925,"tiers":{"easy":92.7645462767561,"standard":85.96308812061348,"judge":84.43983402489627,"hard":52.996474735605155},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":18.673039610051582,"tiers":{"easy":64.70588235294117,"standard":36.36363636363635,"judge":29.41176470588236,"hard":-9.734513274336287},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":62.345713910559354,"tiers":{"easy":32.955006050581126,"standard":73.96017267622133,"judge":63.217955635894754,"hard":63.23198019872138},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":73.08890597287122,"tiers":{"easy":85.3515625,"standard":72.83223907629615,"judge":72.1731448763251,"hard":70.83839611178615},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":10.181818181818176,"tiers":{"easy":43.99999999999999,"standard":-12.0,"judge":19.999999999999996,"hard":5.454545454545445},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":57.23179760285851,"tiers":{"easy":69.61177215806845,"standard":62.693423998098766,"judge":53.08672551735734,"hard":54.51479483056176},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":86.76753584348495,"parts":{"choice":{"ece_hard":0.06104189082173234,"ece_score":87.79162183565353,"mean_tvd":0.2675015176296743,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":80.52073503634305},"noul":{"ece":0.04726762280733596,"brier":0.11782616492468487,"n":495.0,"score":90.54647543853281,"abstention_rate":0.37575757575757573},"score":{"nrps":0.09595485271258676,"top_ece":0.059668603087916515,"n":330.0,"interval90_coverage":0.9333333333333333,"interval90_width":0.4224747474747473,"score":89.235397055579}}},"speed":{"p50_s_raw":0.07847566052805632,"p95_s_raw":0.6763987306097987,"p50_s_adjusted":0.30695132105611267,"p95_s_adjusted":1.5027974612195973,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.042420935960591134,"basis":"documented hosted-model estimate"},"rank":15,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":15,"B":15,"C":19}},{"key":"jevk5-v02","display":"JevK5 v0.2.0","addendum":null,"author":"allebee","repo":"https://github.com/allebee/jevk5","class":"jev-rebuild","licence":"Apache-2.0 (code/adapter); Apache-2.0 (Qwen3.5-4B base)","open":"unknown","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX5090), offline read-only container","gpu":"RTX5090","adapter_id":"images/jevk5-v02/spec.json:jevk5_direct.JevK5DirectAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":46.704186767134146,"calibration":84.87188332789893,"speed":90.85646347150488,"cost":63.07359395281681},"scores":{"A":58.11452490392985,"B":53.55068001120491,"C":40.35730896106238},"views":{"Balanced 33:33:33":54.22450179173148,"Emphasis on Accuracy 60:20:20":47.89028098056264,"Emphasis on Cost 20:20:60":54.54483488475008,"Emphasis on Speed 20:60:20":62.069633409460685,"Intelligence only":40.74998323664581},"composite_ci95":{"A":[47.559542661909845,68.24036195737175],"B":[43.288013328987475,63.73326751910801],"C":[33.027460181881835,47.99518550145878]},"jevbench_score":58.11452490392985,"intelligence":{"I_open":50.008514935705065,"I_sealed":43.39985859856322,"base":46.704186767134146,"gap":6.608656337141845,"excess":1.4220288370626024,"penalty":1.0,"per_type_split":{"open|choice":{"cc":77.60514266103064,"tiers":{"easy":98.19113656918904,"standard":85.96308812061348,"judge":89.6265560165975,"hard":59.263611437524474},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":12.186992254665341,"tiers":{"easy":52.941176470588225,"standard":15.15151515151516,"judge":35.29411764705883,"hard":-16.814159292035402},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":60.23340989141923,"tiers":{"easy":28.60779652081231,"standard":74.75649985421431,"judge":62.07745703499336,"hard":59.49523289499283},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":71.7603783528002,"tiers":{"easy":82.421875,"standard":68.75707493774055,"judge":68.19787985865725,"hard":73.2685297691373},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-3.890909090909095,"tiers":{"easy":43.99999999999999,"standard":-16.000000000000004,"judge":0.0,"hard":-12.727272727272732},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":62.330106533798556,"tiers":{"easy":74.06669119758907,"standard":67.10954514563969,"judge":62.80257002389328,"hard":56.651893444359324},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":84.87188332789893,"parts":{"choice":{"ece_hard":0.05857257507337988,"ece_score":88.28548498532402,"mean_tvd":0.2615167615672721,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":81.0669044142984},"noul":{"ece":0.06765822430583412,"brier":0.11693544190853578,"n":495.0,"score":86.46835513883318,"abstention_rate":0.4242424242424242},"score":{"nrps":0.0947385599255009,"top_ece":0.08182681573159765,"n":330.0,"interval90_coverage":0.9424242424242424,"interval90_width":0.4585858585858586,"score":87.08039043056519}}},"speed":{"p50_s_raw":0.033216340001672506,"p95_s_raw":0.11467095024418086,"p50_s_adjusted":0.216432680003345,"p95_s_adjusted":0.3793419004883617,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.01701689039408867,"basis":"documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)"},"rank":16,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":16,"B":16,"C":22}},{"key":"qwen35-9b-jev-data-mix-v2","display":"Qwen3.5-9B Jev-like data-mix v2","addendum":null,"author":"jsaurabh","repo":"https://huggingface.co/jsaurabh/qwen3.5-9b-jev-data-mix-v2","class":"jev-rebuild","licence":"Apache-2.0 (adapter code and weights; Qwen3.5-9B base is Apache-2.0)","open":"unknown","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/qwen35-9b-jev-data-mix-v2/spec.json:qwen35_nimble_lora.Qwen35NimbleLoraAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":60.448690411881714,"calibration":80.85511159800245,"speed":81.99549950557196,"cost":45.690218065856605},"scores":{"A":53.02414313946459,"B":52.494350100112975,"C":53.02414313946459},"views":{"Balanced 33:33:33":49.483494063390474,"Emphasis on Accuracy 60:20:20":49.87615766345389,"Emphasis on Cost 20:20:60":44.22951871313033,"Emphasis on Speed 20:60:20":55.65672146393036,"Intelligence only":50.476978374194644},"composite_ci95":{"A":[51.87649654035859,53.88222182087126],"B":[50.98151113113002,53.70330555388338],"C":[46.887046030978055,53.88222182087126]},"jevbench_score":53.02414313946459,"intelligence":{"I_open":62.6138776717486,"I_sealed":58.28350315201483,"base":60.448690411881714,"gap":4.330374519733766,"excess":-0.8562529803454764,"penalty":1.0,"per_type_split":{"open|choice":{"cc":72.91194808577592,"tiers":{"easy":98.19113656918904,"standard":82.84377436963868,"judge":87.55186721991701,"hard":50.64629847238542},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":42.36587635858848,"tiers":{"easy":64.70588235294117,"standard":60.606060606060595,"judge":68.62745098039215,"hard":7.964601769911495},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":72.5638085708814,"tiers":{"easy":88.96008632590154,"standard":82.11242483530464,"judge":73.89712850993713,"hard":62.69044104562293},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":74.7723778131831,"tiers":{"easy":82.421875,"standard":79.6241793072221,"judge":77.47349823321555,"hard":68.408262454435},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":31.67272727272728,"tiers":{"easy":43.99999999999999,"standard":12.00000000000001,"judge":32.00000000000001,"hard":38.18181818181819},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":68.40540437013414,"tiers":{"easy":76.24269933016151,"standard":78.78101956011632,"judge":66.09094517211264,"hard":62.99411743365233},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":80.85511159800245,"parts":{"choice":{"ece_hard":0.11208446592845642,"ece_score":77.5831068143087,"mean_tvd":0.310202368163371,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":73.2814349989858},"noul":{"ece":0.08670530591940094,"brier":0.12223515496642685,"n":495.0,"score":82.65893881611981,"abstention_rate":0.23232323232323238},"score":{"nrps":0.07503667025703346,"top_ece":0.09623205508246563,"n":330.0,"interval90_coverage":0.906060606060606,"interval90_width":0.1997474747474748,"score":86.62496097890177}}},"speed":{"p50_s_raw":0.21541375573724508,"p95_s_raw":0.4687169534619888,"p50_s_adjusted":0.5808275114744902,"p95_s_adjusted":1.0874339069239776,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.06461391625615764,"basis":"documented hosted-model estimate"},"rank":17,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":17,"B":17,"C":11}},{"key":"standardone-8b","display":"Standard One 8B (Standard Thinking)","addendum":null,"author":"Standard Thinking (myeongho12)","repo":"https://huggingface.co/StandardThinking/StandardOne-8B","class":"jev-rebuild","licence":"Apache-2.0 (jev-adapter server, LoRA and merged weights; base Ministral 3 8B Apache-2.0)","open":"yes","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX5090), offline read-only container","gpu":"RTX5090","adapter_id":"images/standardone-8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":59.59798327565973,"calibration":83.15842292777755,"speed":92.47668217708318,"cost":43.283250331904235},"scores":{"A":47.79069264973117,"B":47.13021336388438,"C":47.15241622866449},"views":{"Balanced 33:33:33":44.34504647890239,"Emphasis on Accuracy 60:20:20":44.47100595116002,"Emphasis on Cost 20:20:60":38.6660890601248,"Emphasis on Speed 20:60:20":51.807344644154206,"Intelligence only":44.661292576950785},"composite_ci95":{"A":[46.67227934534719,48.54237759923712],"B":[45.66560169780469,48.273280957146895],"C":[40.81880744242586,48.54237759923712]},"jevbench_score":47.79069264973117,"intelligence":{"I_open":59.95626170147317,"I_sealed":59.239704849846284,"base":59.59798327565973,"gap":0.7165568516268834,"excess":-4.470070648452359,"penalty":1.0,"per_type_split":{"open|choice":{"cc":73.71524106786116,"tiers":{"easy":100.0,"standard":82.84377436963868,"judge":89.6265560165975,"hard":50.64629847238542},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":45.04235483412995,"tiers":{"easy":70.58823529411764,"standard":54.54545454545454,"judge":72.54901960784315,"hard":13.27433628318584},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":61.11118920242842,"tiers":{"easy":35.9733866269639,"standard":75.16804575299989,"judge":63.69908035768006,"hard":58.426293204570065},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":72.17810329904881,"tiers":{"easy":88.28125,"standard":79.6241793072221,"judge":66.87279151943464,"hard":68.408262454435},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":42.981818181818184,"tiers":{"easy":60.00000000000001,"standard":39.99999999999999,"judge":36.00000000000001,"hard":45.45454545454546},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":62.55919306867187,"tiers":{"easy":67.88458997347536,"standard":63.500144415233116,"judge":57.21070882928115,"hard":64.7687313487334},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":83.15842292777755,"parts":{"choice":{"ece_hard":0.05968117462976331,"ece_score":88.06376507404734,"mean_tvd":0.24762246288483555,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":81.6507593927819},"noul":{"ece":0.0904581989687968,"brier":0.1212055351648418,"n":495.0,"score":81.90836020624064,"abstention_rate":0.1898989898989899},"score":{"nrps":0.10385562133359677,"top_ece":0.0889106974901002,"n":330.0,"interval90_coverage":0.8787878787878788,"interval90_width":0.28232323232323225,"score":85.91614918431014}}},"speed":{"p50_s_raw":0.026684600859880447,"p95_s_raw":0.06400057689752427,"p50_s_adjusted":0.2033692017197609,"p95_s_adjusted":0.27800115379504853,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.07772456896551724,"basis":"documented hosted-model estimate"},"rank":18,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":18,"B":20,"C":16}},{"key":"ninfer-qwen3.8-flash-next","display":"NInfer Qwen3.8-Flash-Next mixed","addendum":null,"author":"Igor L. / NInfer contributors","repo":"https://github.com/igorls/ninfer","class":"native-logit","licence":"Qwen Community License 1.0 (model); Apache-2.0 (engine)","open":"yes","underlying":"Qwen/Qwen3.8-Flash-Next, mixed NVFP4/FP8/INT4 NInfer artifact","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container","gpu":"RTXPRO6000","adapter_id":"images/ninfer-qwen3.8-flash-next/spec.json:ninfer_native.NInferNativeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":67.19957888400617,"calibration":88.54350643788759,"speed":88.61299548829486,"cost":42.53362878447125},"scores":{"A":47.474808502448944,"B":47.7011582677789,"C":47.474808502448944},"views":{"Balanced 33:33:33":43.7010429741045,"Emphasis on Accuracy 60:20:20":45.54715488980916,"Emphasis on Cost 20:20:60":37.41753230615858,"Emphasis on Speed 20:60:20":50.08128343228141,"Intelligence only":48.62856070723032},"composite_ci95":{"A":[46.69503698220878,47.881123014444135],"B":[46.62486661589989,48.43899187781872],"C":[46.69503698220878,47.881123014444135]},"jevbench_score":47.474808502448944,"intelligence":{"I_open":70.25732348162926,"I_sealed":64.14183428638309,"base":67.19957888400617,"gap":6.11548919524617,"excess":0.9288616951669271,"penalty":1.0,"per_type_split":{"open|choice":{"cc":84.92619742669542,"tiers":{"easy":96.38227313837805,"standard":92.20171562256304,"judge":95.850622406639,"hard":70.23110066588326},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":51.40125881406464,"tiers":{"easy":82.35294117647058,"standard":72.72727272727273,"judge":58.823529411764696,"hard":27.43362831858407},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":74.44451420412774,"tiers":{"easy":59.42108431739883,"standard":88.84655227920712,"judge":71.8468031376396,"hard":72.9476359381364},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":79.10854505246607,"tiers":{"easy":94.140625,"standard":85.05773149196287,"judge":68.19787985865725,"hard":80.55893074119078},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":43.01818181818182,"tiers":{"easy":52.0,"standard":36.00000000000001,"judge":56.00000000000001,"hard":34.54545454545455},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":70.29877598850139,"tiers":{"easy":76.762949788409,"standard":74.31336416172283,"judge":71.11674673491447,"hard":66.06196039210394},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":88.54350643788759,"parts":{"choice":{"ece_hard":0.05943674097877588,"ece_score":88.11265180424482,"mean_tvd":0.2663371154061145,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":80.73947013181669},"noul":{"ece":0.032862434581237034,"brier":0.06073012931047522,"n":495.0,"score":93.42751308375259,"abstention_rate":0.2404040404040404},"score":{"nrps":0.062258834532185996,"top_ece":0.054235221752971935,"n":330.0,"interval90_coverage":0.9818181818181818,"interval90_width":0.34166666666666656,"score":91.4635360980935}}},"speed":{"p50_s_raw":0.07975747575983405,"p95_s_raw":0.14732526696752724,"p50_s_adjusted":0.3095149515196681,"p95_s_adjusted":0.4446505339350545,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.08232764778325123,"basis":"documented hosted-model estimate"},"rank":19,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":19,"B":18,"C":15}},{"key":"swanone","display":"swanOne (blockbrain, Qwen3.8-Flash-Next NVFP4)","addendum":null,"author":"blockbrain","repo":"https://github.com/blockbrain-ai/swanone-recipe","class":"system-one-open","licence":"patches Apache-2.0, shim MIT; weights under the Qwen licence (LICENSE-NOTICE.md)","open":"yes","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container","gpu":"RTXPRO6000","adapter_id":"images/swanone/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":71.22352028588858,"calibration":87.05674154866381,"speed":84.74079430681516,"cost":42.15074720483203},"scores":{"A":46.56179491157818,"B":47.31995862079193,"C":46.56179491157818},"views":{"Balanced 33:33:33":43.014358880893774,"Emphasis on Accuracy 60:20:20":45.76374642916793,"Emphasis on Cost 20:20:60":36.627361387652215,"Emphasis on Speed 20:60:20":48.56537743728146,"Intelligence only":50.61671801334552},"composite_ci95":{"A":[45.81356430623499,46.94332947688013],"B":[46.35320829142545,48.000773651251464],"C":[45.81356430623499,46.94332947688013]},"jevbench_score":46.56179491157818,"intelligence":{"I_open":70.49798699750956,"I_sealed":71.9490535742676,"base":71.22352028588858,"gap":-1.4510665767580377,"excess":-6.63769407683728,"penalty":1.0,"per_type_split":{"open|choice":{"cc":88.10818162872131,"tiers":{"easy":100.0,"standard":92.20171562256304,"judge":95.850622406639,"hard":77.2816294555425},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":45.54950862082565,"tiers":{"easy":76.47058823529412,"standard":57.57575757575757,"judge":49.01960784313726,"hard":29.20353982300885},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":77.83627074298171,"tiers":{"easy":65.05719927474456,"standard":89.87197767107257,"judge":74.955420484181,"hard":77.1738228400961},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":82.64893490554081,"tiers":{"easy":94.140625,"standard":86.41611953814805,"judge":77.47349823321555,"hard":81.77399756986635},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":59.89090909090909,"tiers":{"easy":92.0,"standard":52.0,"judge":63.99999999999999,"hard":52.727272727272734},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":73.30731672635291,"tiers":{"easy":84.42086253601506,"standard":74.97120272471354,"judge":71.05934999875308,"hard":71.38296232045693},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":87.05674154866381,"parts":{"choice":{"ece_hard":0.028102447139622583,"ece_score":94.37951057207549,"mean_tvd":0.30445967398626356,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":81.96677158672458},"noul":{"ece":0.06737763907474524,"brier":0.08484464490362868,"n":495.0,"score":86.52447218505095,"abstention_rate":0.19999999999999996},"score":{"nrps":0.05807570109811426,"top_ece":0.0441723407087838,"n":330.0,"interval90_coverage":0.9787878787878788,"interval90_width":0.25883838383838387,"score":92.67898087421591}}},"speed":{"p50_s_raw":0.2087587924906984,"p95_s_raw":0.22074080496095116,"p50_s_adjusted":0.5675175849813968,"p95_s_adjusted":0.5914816099219024,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.08478293103448274,"basis":"price floor: base-model reference price applied"},"rank":20,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":20,"B":19,"C":18}},{"key":"raw-qwen3-8b","display":"Raw Qwen3 8B direct logits","addendum":null,"author":"Alibaba Qwen / neutral reproduction","repo":"https://huggingface.co/Qwen/Qwen3-8B","class":"raw-logit-control","licence":"Apache-2.0","open":"yes","underlying":"Qwen3-8B BF16","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/raw-qwen3-8b/spec.json:raw_adapter.RawControlAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":51.1392760971673,"calibration":49.19480442556255,"speed":86.84465631896776,"cost":45.53515592313027},"scores":{"A":45.223917256408555,"B":44.632516465694714,"C":32.85298453350857},"views":{"Balanced 33:33:33":46.9192160309914,"Emphasis on Accuracy 60:20:20":45.0069155506764,"Emphasis on Cost 20:20:60":42.77253755222896,"Emphasis on Speed 20:60:20":54.52148806396662,"Intelligence only":42.41390150199564},"composite_ci95":{"A":[36.887258965736855,46.74342614714434],"B":[36.02835612521688,46.43665511058838],"C":[25.616152059539484,38.86132200528848]},"jevbench_score":45.223917256408555,"intelligence":{"I_open":56.77526351111453,"I_sealed":45.503288683220056,"base":51.1392760971673,"gap":11.271974827894475,"excess":6.085347327815232,"penalty":1.0,"per_type_split":{"open|choice":{"cc":64.85654971862496,"tiers":{"easy":92.7645462767561,"standard":71.92617624122693,"judge":88.58921161825727,"hard":36.54524089306697},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":48.067609988484534,"tiers":{"easy":82.35294117647058,"standard":48.484848484848484,"judge":80.3921568627451,"hard":15.04424778761062},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":57.40163082623409,"tiers":{"easy":47.131947467639016,"standard":77.03790981221292,"judge":53.06730608793813,"hard":53.40165572661542},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":56.54764008742353,"tiers":{"easy":73.63281249999999,"standard":64.68191079918498,"judge":52.296819787985875,"hard":51.39732685297693},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":38.4,"tiers":{"easy":60.00000000000001,"standard":43.99999999999999,"judge":52.0,"hard":19.999999999999996},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":41.56222596223666,"tiers":{"easy":56.8339149235689,"standard":42.20719309928661,"judge":29.4866104671199,"hard":46.47853177471619},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":49.19480442556255,"parts":{"choice":{"ece_hard":0.3824646871164692,"ece_score":23.50706257670616,"mean_tvd":0.4363539849414789,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":39.935832041279134},"noul":{"ece":0.2219333154858661,"brier":0.2249852735368055,"n":495.0,"score":55.61333690282678,"abstention_rate":0.04040404040404044},"score":{"nrps":0.17281393617880628,"top_ece":0.3932405885847793,"n":330.0,"interval90_coverage":0.6393939393939394,"interval90_width":0.12853535353535353,"score":52.03524433258176}}},"speed":{"p50_s_raw":0.09141041804105043,"p95_s_raw":0.23566609979607117,"p50_s_adjusted":0.3328208360821009,"p95_s_adjusted":0.6213321995921424,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.0653875104679803,"basis":"documented hosted-model estimate"},"rank":21,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":21,"B":21,"C":24}},{"key":"decider-2b","display":"decider-2b (Mapika)","addendum":null,"author":"Mapika","repo":"https://huggingface.co/Mapika/decider-2b","class":"jev-rebuild","licence":"Apache-2.0","open":"yes","underlying":"Qwen3.5-2B-Base with a trained decision readout, 1.9B","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/decider-2b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":42.33871878833063,"calibration":71.52659723196619,"speed":94.40334833224873,"cost":64.91766177740826},"scores":{"A":45.09827651852322,"B":41.10644266575027,"C":31.318247582307787},"views":{"Balanced 33:33:33":43.35456011050343,"Emphasis on Accuracy 60:20:20":37.015822693274224,"Emphasis on Cost 20:20:60":44.5777637422437,"Emphasis on Speed 20:60:20":50.6362696545408,"Intelligence only":30.357997888722927},"composite_ci95":{"A":[33.81613238500261,54.263223032815134],"B":[30.279563783540407,50.04341291318229],"C":[23.48342526736293,37.68279377278829]},"jevbench_score":45.09827651852322,"intelligence":{"I_open":48.790563557043,"I_sealed":35.88687401961826,"base":42.33871878833063,"gap":12.90368953742474,"excess":7.717062037345497,"penalty":1.0,"per_type_split":{"open|choice":{"cc":52.01934595406228,"tiers":{"easy":89.14681941513415,"standard":68.80686249025213,"judge":61.6182572614108,"hard":27.144535840188002},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":35.238117773255716,"tiers":{"easy":82.35294117647058,"standard":30.303030303030297,"judge":68.62745098039215,"hard":0.8849557522123908},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":59.114226943811005,"tiers":{"easy":26.393778801843315,"standard":74.94920494699646,"judge":59.75314547837483,"hard":58.8976610767873},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":53.48470768046778,"tiers":{"easy":79.4921875,"standard":59.2483586144442,"judge":46.99646643109541,"hard":48.96719319562577},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":5.7818181818181795,"tiers":{"easy":36.00000000000001,"standard":0.0,"judge":0.0,"hard":5.454545454545445},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":48.39409619656884,"tiers":{"easy":73.47898550724638,"standard":60.693366918555846,"judge":39.21284751474303,"hard":42.8591750192753},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":71.52659723196619,"parts":{"choice":{"ece_hard":0.2555180505415162,"ece_score":48.89638989169676,"mean_tvd":0.38142566660000665,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":55.376911615848044},"noul":{"ece":0.14005999999999988,"brier":0.18936641214141414,"n":495.0,"score":71.98800000000003,"abstention_rate":0.22424242424242424},"score":{"nrps":0.11813754991414144,"top_ece":0.06878242424242426,"n":330.0,"interval90_coverage":0.9030303030303031,"interval90_width":0.4565656565656566,"score":87.2148800800505}}},"speed":{"p50_s_raw":0.01366500393487513,"p95_s_raw":0.02729465400916524,"p50_s_adjusted":0.17733000786975026,"p95_s_adjusted":0.20458930801833047,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.014771046798029557,"basis":"documented hosted-model estimate"},"rank":22,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":22,"B":26,"C":26}},{"key":"system-one-sg","display":"system-one (Qwen3-8B, Sean Goedecke)","addendum":null,"author":"Sean Goedecke","repo":"https://github.com/sgoedecke/system-one","class":"jev-rebuild","licence":"no licence file in the repository as of 19 Sep; Qwen3 weights Apache-2.0","open":"yes","underlying":"Qwen/Qwen3-8B (frozen, BF16), the model of the author's demos","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX5090), offline read-only container","gpu":"RTX5090","adapter_id":"images/system-one-sg/spec.json:remote_inproc_v15.RemoteInprocV15Adapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":50.47245577952833,"calibration":49.37850478055448,"speed":90.87700150484226,"cost":44.96855803039237},"scores":{"A":44.13645499190376,"B":43.43200453121163,"C":31.232289361046593},"views":{"Balanced 33:33:33":45.73803572130066,"Emphasis on Accuracy 60:20:20":43.637702179733125,"Emphasis on Cost 20:20:60":41.46769673439668,"Emphasis on Speed 20:60:20":53.879881804933696,"Intelligence only":40.82557881705132},"composite_ci95":{"A":[36.78500148194342,45.662322394284914],"B":[35.915897669029526,45.22132798708415],"C":[25.545139918016265,36.98532780405549]},"jevbench_score":44.13645499190376,"intelligence":{"I_open":53.53755450895149,"I_sealed":47.40735705010517,"base":50.47245577952833,"gap":6.130197458846318,"excess":0.9435699587670756,"penalty":1.0,"per_type_split":{"open|choice":{"cc":63.17740474707069,"tiers":{"easy":94.57340970756708,"standard":73.48583311671433,"judge":78.21576763485477,"hard":38.895417156286705},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":49.88579180666636,"tiers":{"easy":100.0,"standard":57.57575757575757,"judge":74.50980392156863,"hard":15.04424778761062},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":47.54946697311742,"tiers":{"easy":22.44824791409641,"standard":72.50561783397852,"judge":42.37602381862431,"hard":45.22677867331196},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":56.6787225986529,"tiers":{"easy":79.4921875,"standard":61.96513470681459,"judge":50.97173144876326,"hard":52.61239368165249},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":40.03636363636364,"tiers":{"easy":43.99999999999999,"standard":19.999999999999996,"judge":39.99999999999999,"hard":49.09090909090909},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":45.50698491529897,"tiers":{"easy":67.19757471079686,"standard":60.6717253542459,"judge":36.61887846690121,"hard":39.16804708324935},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":49.37850478055448,"parts":{"choice":{"ece_hard":0.3414165000468772,"ece_score":31.716699990624562,"mean_tvd":0.43927763866315545,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":43.894468062154516},"noul":{"ece":0.22457895725345312,"brier":0.22985988066791246,"n":495.0,"score":55.08420854930938,"abstention_rate":0.04040404040404044},"score":{"nrps":0.17575051481605997,"top_ece":0.42055636528997464,"n":330.0,"interval90_coverage":0.5787878787878787,"interval90_width":0.09949494949494948,"score":49.15683773019954}}},"speed":{"p50_s_raw":0.0384387425146997,"p95_s_raw":0.10508537914138283,"p50_s_adjusted":0.2268774850293994,"p95_s_adjusted":0.3601707582827657,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.06829381896551724,"basis":"documented hosted-model estimate"},"rank":23,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":23,"B":22,"C":27}},{"key":"system-one-open","display":"system-one-open (Gemma 4 E2B LoRA on an L4)","addendum":null,"author":"mithalouni","repo":"https://github.com/mithalouni/system-one-open","class":"jev-rebuild","licence":"MIT (repository LICENSE; Gemma weights keep Google’s terms)","open":"yes","underlying":"google/gemma-4-E2B-it + LoRA","endpoint_kind":"demo","endpoint_condition":"the author's public demo endpoint","gpu":null,"adapter_id":"harness/specs/system-one-open.json:jevbench.adapters.systemone_list.SystemOneListAdapter","api_flag":true,"api_exposure_note":"API measurement: the operator's endpoint received sealed item text, without answers.","status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":41.64356524223511,"calibration":71.62757593487629,"speed":78.27072904753155,"cost":68.33006522881325},"scores":{"A":42.43603455732882,"B":38.796663536397624,"C":29.469468442589456},"views":{"Balanced 33:33:33":40.46769109259382,"Emphasis on Accuracy 60:20:20":34.87521081607888,"Emphasis on Cost 20:20:60":42.98178498751218,"Emphasis on Speed 20:60:20":45.05746080516129,"Intelligence only":28.887083896478522},"composite_ci95":{"A":[33.838584413879616,51.9229885095774],"B":[30.532950806264438,48.14184520284253],"C":[23.499016954083064,36.05763090942875]},"jevbench_score":42.43603455732882,"intelligence":{"I_open":44.43803383833533,"I_sealed":38.849096646134896,"base":41.64356524223511,"gap":5.588937192200433,"excess":0.4023096921211904,"penalty":1.0,"per_type_split":{"open|choice":{"cc":59.92067737087481,"tiers":{"easy":96.38227313837805,"standard":70.36651936573953,"judge":75.10373443983403,"hard":34.19506462984723},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":15.40091177259319,"tiers":{"easy":76.47058823529412,"standard":39.393939393939405,"judge":7.843137254901955,"hard":-6.194690265486724},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":57.992512371538,"tiers":{"easy":41.64728311554955,"standard":71.89847009741732,"judge":63.21182539395285,"hard":51.21135605578431},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":52.042685008120415,"tiers":{"easy":76.56249999999999,"standard":59.2483586144442,"judge":49.64664310954064,"hard":44.10692588092346},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":14.981818181818177,"tiers":{"easy":36.00000000000001,"standard":15.999999999999993,"judge":19.999999999999996,"hard":5.454545454545445},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":49.5227867484661,"tiers":{"easy":61.75933432475533,"standard":61.00129485100315,"judge":40.54257912619954,"hard":47.45955151982518},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":71.62757593487629,"parts":{"choice":{"ece_hard":0.19489049120715382,"ece_score":61.02190175856923,"mean_tvd":0.3319398583447739,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":63.91395796204592},"noul":{"ece":0.12032569308344548,"brier":0.17386092858386082,"n":495.0,"score":75.9348613833109,"abstention_rate":0.2848484848484848},"score":{"nrps":0.12647097490053033,"top_ece":0.18642542795701458,"n":330.0,"interval90_coverage":0.8151515151515152,"interval90_width":0.3361111111111109,"score":75.03390845927203}}},"speed":{"p50_s_raw":0.5750190168619156,"p95_s_raw":0.647418188303709,"p50_s_adjusted":1.1500380337238312,"p95_s_adjusted":1.294836376607418,"n":450,"adjustment":"x2 demo-endpoint adjustment (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.011367487684729064,"basis":"ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies"},"rank":24,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":24,"B":27,"C":29}},{"key":"kushal-gemma4-31b-it-autoloops","display":"Autoloops – Gemma 4 31B IT","addendum":null,"author":"Autoloops","repo":null,"class":"jev","licence":"Gemma terms of use (Google) for the base weights; hosted Autoloops service","open":"no","underlying":null,"endpoint_kind":"api","endpoint_condition":"the operator's hosted API","gpu":null,"adapter_id":"harness/specs/kushal-gemma4-31b-it-autoloops.json:extra.autoloops_systemone.AutoloopsSystemOneAdapter","api_flag":true,"api_exposure_note":"API measurement: the operator's endpoint received sealed item text, without answers.","status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":76.68463397202072,"calibration":85.84316449763286,"speed":83.9240668357571,"cost":39.586359989532696},"scores":{"A":40.52814707638268,"B":41.84081656494916,"C":40.52814707638268},"views":{"Balanced 33:33:33":37.44726037707129,"Emphasis on Accuracy 60:20:20":41.07786216973585,"Emphasis on Cost 20:20:60":31.111486702915485,"Emphasis on Speed 20:60:20":42.32591427429326,"Intelligence only":48.06837932931786},"composite_ci95":{"A":[40.08903740319995,40.76240789696144],"B":[41.21534979263599,42.28834759900149],"C":[40.08903740319995,40.76240789696144]},"jevbench_score":40.52814707638268,"intelligence":{"I_open":76.87888067003877,"I_sealed":76.49038727400266,"base":76.68463397202072,"gap":0.388493396036111,"excess":-4.798134104043132,"penalty":1.0,"per_type_split":{"open|choice":{"cc":86.35693213456801,"tiers":{"easy":98.19113656918904,"standard":93.76137249805042,"judge":96.88796680497926,"hard":71.7978848413631},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":68.73077469121196,"tiers":{"easy":76.47058823529412,"standard":72.72727272727273,"judge":78.43137254901961,"hard":57.52212389380531},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":75.54893518433636,"tiers":{"easy":79.44374053586364,"standard":84.43635298314938,"judge":75.86669966490899,"hard":69.89320158661853},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":87.5068098550186,"tiers":{"easy":100.0,"standard":83.69934344577767,"judge":85.42402826855124,"hard":87.84933171324423},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":67.45454545454545,"tiers":{"easy":100.0,"standard":63.99999999999999,"judge":63.99999999999999,"hard":63.63636363636365},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":74.50980651244393,"tiers":{"easy":77.31828535179173,"standard":74.77474662968355,"judge":74.55125067882011,"hard":73.64413361920504},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":85.84316449763286,"parts":{"choice":{"ece_hard":0.11069466505831069,"ece_score":77.86106698833785,"mean_tvd":0.24261659079045955,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":76.79970395464595},"noul":{"ece":0.024641134940521763,"brier":0.04314422942385484,"n":495.0,"score":95.07177301189564,"abstention_rate":0.13535353535353534},"score":{"nrps":0.05012237564914895,"top_ece":0.11835864691185555,"n":330.0,"interval90_coverage":0.9696969696969697,"interval90_width":0.33282828282828275,"score":85.658016526357}}},"speed":{"p50_s_raw":0.6074120700359344,"p95_s_raw":0.6669755268841981,"p50_s_adjusted":0.6074120700359344,"p95_s_adjusted":0.6669755268841981,"n":450,"adjustment":"none (hosted API, measured as is)"},"cost":{"kind":"tariff","usd_per_1000":0.10322573891625615,"basis":"operator standard launch list price (interpretation I-1)"},"rank":25,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":25,"B":24,"C":20}},{"key":"gpt-6-luna-low","display":"GPT-6 Luna (low reasoning effort)","addendum":null,"author":"OpenAI","repo":null,"class":"llm-baseline","licence":"proprietary API","open":"unknown","underlying":null,"endpoint_kind":"api","endpoint_condition":"the operator's hosted API","gpu":null,"adapter_id":"harness/specs/gpt-6-luna-low.json:jevbench.adapters.openai_compat.OpenAICompatAdapter","api_flag":true,"api_exposure_note":"API measurement: the operator's endpoint received sealed item text, without answers.","status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":95.29107650041868,"calibration":94.92401460047215,"speed":73.19737721417388,"cost":39.05771622064802},"scores":{"A":40.481958334046816,"B":43.10073731729786,"C":40.481958334046816},"views":{"Balanced 33:33:33":36.78945782049289,"Emphasis on Accuracy 60:20:20":43.125459438148994,"Emphasis on Cost 20:20:60":30.21841757014399,"Emphasis on Speed 20:60:20":39.58116868050534,"Intelligence only":58.146812947733785},"composite_ci95":{"A":[40.268976996595626,40.66385155794427],"B":[42.806610244055186,43.35101400348243],"C":[40.268976996595626,40.66385155794427]},"jevbench_score":40.481958334046816,"intelligence":{"I_open":94.72456284907499,"I_sealed":95.8575901517624,"base":95.29107650041868,"gap":-1.1330273026874096,"excess":-6.319654802766652,"penalty":1.0,"per_type_split":{"open|choice":{"cc":99.0613549547106,"tiers":{"easy":100.0,"standard":98.44034312451261,"judge":100.0,"hard":98.43321582452018},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":85.24505860268484,"tiers":{"easy":100.0,"standard":96.96969696969697,"judge":88.23529411764706,"hard":73.45132743362832},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":99.86727498982954,"tiers":{"easy":99.87903225806451,"standard":99.90724381625442,"judge":99.96068152031454,"hard":99.77429636169461},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":97.87271569458548,"tiers":{"easy":100.0,"standard":98.64161195381482,"judge":98.67491166077738,"hard":96.35479951397326},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":90.18181818181819,"tiers":{"easy":100.0,"standard":92.0,"judge":92.0,"hard":85.45454545454545},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":99.51823657888352,"tiers":{"easy":99.90942028985508,"standard":99.93073047858942,"judge":98.60362257792755,"hard":99.90015420200463},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":94.92401460047215,"parts":{"choice":{"ece_hard":0.04035662449133609,"ece_score":91.92867510173278,"mean_tvd":0.023317194080665005,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":94.79847784683314},"noul":{"ece":0.04876064279151522,"brier":0.04657367170495567,"n":495.0,"score":90.24787144169696,"abstention_rate":0.014141414141414121},"score":{"nrps":0.0008295440859084673,"top_ece":0.0023282828281823414,"n":330.0,"interval90_coverage":0.996969696969697,"interval90_width":0.00202020202020202,"score":99.72569451288635}}},"speed":{"p50_s_raw":1.5798842646181583,"p95_s_raw":3.0313564356416416,"p50_s_adjusted":1.5798842646181583,"p95_s_adjusted":3.0313564356416416,"n":450,"adjustment":"none (hosted API, measured as is)"},"cost":{"kind":"estimate","usd_per_1000":0.10750024630541874,"basis":"operator standard launch list price (interpretation I-1); no exact base-model floor applies"},"rank":26,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":26,"B":23,"C":21}},{"key":"gpt-6-luna","display":"GPT-6 Luna (default medium reasoning effort)","addendum":null,"author":"OpenAI","repo":null,"class":"llm-baseline","licence":"proprietary API","open":"unknown","underlying":null,"endpoint_kind":"api","endpoint_condition":"the operator's hosted API","gpu":null,"adapter_id":"harness/specs/gpt-6-luna.json:jevbench.adapters.openai_compat.OpenAICompatAdapter","api_flag":true,"api_exposure_note":"API measurement: the operator's endpoint received sealed item text, without answers.","status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":96.23569044084438,"calibration":95.56003846585782,"speed":73.18530319155485,"cost":38.316528790869064},"scores":{"A":38.751615238726444,"B":41.35111355834929,"C":38.751615238726444},"views":{"Balanced 33:33:33":35.127911556506724,"Emphasis on Accuracy 60:20:20":41.393941998009545,"Emphasis on Cost 20:20:60":28.688855301298666,"Emphasis on Speed 20:60:20":37.897006955989184,"Intelligence only":56.51561710716807},"composite_ci95":{"A":[38.56772553955819,38.92820298954891],"B":[41.101933604560855,41.59723975450365],"C":[38.56772553955819,38.92820298954891]},"jevbench_score":38.751615238726444,"intelligence":{"I_open":96.31846029492732,"I_sealed":96.15292058676144,"base":96.23569044084438,"gap":0.16553970816588048,"excess":-5.021087791913362,"penalty":1.0,"per_type_split":{"open|choice":{"cc":99.37543984540196,"tiers":{"easy":100.0,"standard":100.0,"judge":98.96265560165975,"hard":99.21660791226009},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":90.68714211348257,"tiers":{"easy":100.0,"standard":100.0,"judge":90.19607843137254,"hard":84.070796460177},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":98.89279892589744,"tiers":{"easy":100.0,"standard":99.886925795053,"judge":99.89777195281782,"hard":97.3652054526037},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":99.51397326852977,"tiers":{"easy":100.0,"standard":100.0,"judge":100.00000000000001,"hard":98.78493317132441},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":89.92727272727274,"tiers":{"easy":92.0,"standard":96.0,"judge":96.0,"hard":81.81818181818181},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":99.01751576448186,"tiers":{"easy":99.9375,"standard":99.8942065491184,"judge":98.51263689974726,"hard":98.727833461835},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":95.56003846585782,"parts":{"choice":{"ece_hard":0.03585405941841228,"ece_score":92.82918811631754,"mean_tvd":0.031053666112664996,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":94.86191075252552},"noul":{"ece":0.03508381739474774,"brier":0.03327200606740464,"n":495.0,"score":92.98323652105044,"abstention_rate":0.012121212121212088},"score":{"nrps":0.0028561930758074566,"top_ece":0.010222222222121494,"n":330.0,"interval90_coverage":0.990909090909091,"interval90_width":0.0027777777777777775,"score":98.83496812399747}}},"speed":{"p50_s_raw":1.5554233863949776,"p95_s_raw":3.0876001715660073,"p50_s_adjusted":1.5554233863949776,"p95_s_adjusted":3.0876001715660073,"n":450,"adjustment":"none (hosted API, measured as is)"},"cost":{"kind":"estimate","usd_per_1000":0.1137930418719212,"basis":"operator standard launch list price (interpretation I-1); no exact base-model floor applies"},"rank":27,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":27,"B":25,"C":23}},{"key":"jevone","display":"JevOne","addendum":null,"author":"Juspay","repo":"https://huggingface.co/juspay/jev-one","class":"jev-rebuild","licence":"Component-specific JevOne/Qwen/SGLang terms recorded in RESULT.md","open":"yes","underlying":"Qwen3.6-35B-A3B BF16 with JevOne bidirectional option-logit mapping","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container","gpu":"RTXPRO6000","adapter_id":"images/jevone/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":54.13082284597005,"calibration":84.94489158761552,"speed":90.43197361602677,"cost":39.84212731591276},"scores":{"A":38.24864204152178,"B":37.40461666043465,"C":31.131695776173597},"views":{"Balanced 33:33:33":34.86809688509843,"Emphasis on Accuracy 60:20:20":34.66746178277511,"Emphasis on Cost 20:20:60":30.28542415522889,"Emphasis on Speed 20:60:20":41.366998522047716,"Intelligence only":34.37080137397821},"composite_ci95":{"A":[37.37790294644138,38.7857176460762],"B":[36.19750772097442,38.28517425236358],"C":[26.636189982602396,35.41851297457285]},"jevbench_score":38.24864204152178,"intelligence":{"I_open":53.90312873058646,"I_sealed":54.358516961353644,"base":54.13082284597005,"gap":-0.45538823076718415,"excess":-5.642015730846427,"penalty":1.0,"per_type_split":{"open|choice":{"cc":82.42577264151704,"tiers":{"easy":96.38227313837805,"standard":87.52274499610087,"judge":94.81327800829875,"hard":67.09753231492361},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":9.375956335873049,"tiers":{"easy":64.70588235294117,"standard":15.15151515151516,"judge":7.843137254901955,"hard":-6.194690265486724},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":69.9076572143693,"tiers":{"easy":40.016129032258064,"standard":83.07703180212015,"judge":72.52889908256881,"hard":68.82992056487203},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":77.14053600029337,"tiers":{"easy":91.2109375,"standard":76.9074032148517,"judge":76.14840989399293,"hard":74.48359659781289},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":17.127272727272725,"tiers":{"easy":68.0,"standard":0.0,"judge":32.00000000000001,"hard":1.8181818181818077},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":68.80774215649485,"tiers":{"easy":81.35172101449275,"standard":72.98148614609572,"judge":64.51078348778432,"hard":66.80759444872784},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":84.94489158761552,"parts":{"choice":{"ece_hard":0.032576534296028775,"ece_score":93.48469314079423,"mean_tvd":0.30537816655000166,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":81.47343824289703},"noul":{"ece":0.07786626262626258,"brier":0.09677940056565656,"n":495.0,"score":84.42674747474749,"abstention_rate":0.42020202020202024},"score":{"nrps":0.07153506758080812,"top_ece":0.07488757575757568,"n":330.0,"interval90_coverage":0.9484848484848485,"interval90_width":0.33308080808080787,"score":88.93448904520203}}},"speed":{"p50_s_raw":0.05845375219359994,"p95_s_raw":0.09459453681483865,"p50_s_adjusted":0.2669075043871999,"p95_s_adjusted":0.3391890736296773,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.10121908866995075,"basis":"documented hosted-model estimate"},"rank":28,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":28,"B":28,"C":28}},{"key":"kev-4b","display":"kev 4B (research preview)","addendum":null,"author":"Jared Palmer","repo":"https://github.com/jaredpalmer/kev","class":"jev-rebuild","licence":"Apache-2.0","open":true,"underlying":"Qwen3-4B-Base + LoRA + learned pointer head; jaredpalmer/kev-4b","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/kev-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":39.86088085724986,"calibration":67.64154808465537,"speed":85.47775575216679,"cost":65.78036497095258},"scores":{"A":38.073131188431404,"B":34.59396156532314,"C":26.439674436410698},"views":{"Balanced 33:33:33":36.67493457142576,"Emphasis on Accuracy 60:20:20":31.10505213714635,"Emphasis on Cost 20:20:60":38.56878935020766,"Emphasis on Speed 20:60:20":42.15332921905585,"Intelligence only":25.33381916742963},"composite_ci95":{"A":[27.476430952821357,46.767301073415496],"B":[24.5320197349573,43.08613240484619],"C":[19.080854828348166,32.477292412094094]},"jevbench_score":38.073131188431404,"intelligence":{"I_open":47.060039168272525,"I_sealed":33.20122670594433,"base":40.13063293710843,"gap":13.858812462328196,"excess":8.672184962248952,"penalty":0.9932781503775104,"per_type_split":{"open|choice":{"cc":53.88729392667013,"tiers":{"easy":89.14681941513415,"standard":70.36651936573953,"judge":66.80497925311204,"hard":27.144535840188002},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":29.05746691275062,"tiers":{"easy":64.70588235294117,"standard":42.42424242424243,"judge":64.70588235294117,"hard":-13.27433628318584},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":58.23535666539682,"tiers":{"easy":33.1008573207551,"standard":69.32495502200398,"judge":59.779114215484476,"hard":57.81636416068792},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":46.761713448034214,"tiers":{"easy":85.3515625,"standard":55.17319447588863,"judge":35.07067137809188,"hard":41.67679222357231},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":4.2909090909090875,"tiers":{"easy":12.00000000000001,"standard":-4.0000000000000036,"judge":-4.0000000000000036,"hard":12.72727272727272},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":48.55105757888969,"tiers":{"easy":69.97776498664359,"standard":55.93544744902701,"judge":42.91280939382903,"hard":43.73087193067805},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.003754693366708417},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.00303030303030305}},"calibration":{"score":67.64154808465537,"parts":{"choice":{"ece_hard":0.2777823341900978,"ece_score":44.44353316198044,"mean_tvd":0.3226589404273894,"n_tvd":50.0,"n":796.0,"n_ece_hard":277.0,"score":56.08881955962075},"noul":{"ece":0.1829696969696968,"brier":0.22143454545454574,"n":495.0,"score":63.40606060606064,"abstention_rate":0.20404040404040402},"score":{"nrps":0.11772776999236903,"top_ece":0.10683847412096825,"n":329.0,"interval90_coverage":0.9027355623100304,"interval90_width":0.4435157041540021,"score":83.42976408828473}}},"speed":{"p50_s_raw":0.1715529104694724,"p95_s_raw":0.21224620514549314,"p50_s_adjusted":0.49310582093894484,"p95_s_adjusted":0.5744924102909863,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.013824661330049262,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":29,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":29,"B":29,"C":32}},{"key":"kev-8b","display":"kev 8B (research preview)","addendum":null,"author":"Jared Palmer","repo":"https://github.com/jaredpalmer/kev","class":"jev-rebuild","licence":"Apache-2.0","open":true,"underlying":"Qwen3-8B-Base + LoRA + learned pointer head; jaredpalmer/kev-8b","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/kev-8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":48.295292360614766,"calibration":71.01729114829811,"speed":84.14101453293591,"cost":40.42054386073457},"scores":{"A":34.15264116882949,"B":33.094884395798175,"C":23.717111922798257},"views":{"Balanced 33:33:33":31.905696022137963,"Emphasis on Accuracy 60:20:20":30.874469599373526,"Emphasis on Cost 20:20:60":28.542378501445935,"Emphasis on Speed 20:60:20":37.590798427003946,"Intelligence only":29.446839307001515},"composite_ci95":{"A":[26.78663166842091,37.48283011878216],"B":[25.56048047867171,36.685764855628186],"C":[18.601827547514524,27.88353014576553]},"jevbench_score":34.15264116882949,"intelligence":{"I_open":54.3060710172369,"I_sealed":42.28451370399262,"base":48.295292360614766,"gap":12.02155731324428,"excess":6.834929813165037,"penalty":1.0,"per_type_split":{"open|choice":{"cc":60.672703581896855,"tiers":{"easy":94.57340970756708,"standard":73.48583311671433,"judge":77.17842323651452,"hard":33.41167254210731},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":38.20216743173536,"tiers":{"easy":76.47058823529412,"standard":54.54545454545454,"judge":66.66666666666667,"hard":-0.8849557522123908},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":64.04334203807852,"tiers":{"easy":49.42575709183821,"standard":73.94927434439558,"judge":65.26071217868838,"hard":61.831744516022646},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":58.716195357778545,"tiers":{"easy":76.56249999999999,"standard":68.75707493774055,"judge":50.97173144876326,"hard":55.04252733900366},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":14.000000000000004,"tiers":{"easy":36.00000000000001,"standard":0.0,"judge":8.000000000000007,"hard":19.999999999999996},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":54.13734575419933,"tiers":{"easy":68.61186209200632,"standard":57.078521488434596,"judge":49.05877571750435,"hard":52.85705633015117},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0012515644555695093},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.00303030303030305}},"calibration":{"score":71.01729114829811,"parts":{"choice":{"ece_hard":0.226086608660866,"ece_score":54.7826782678268,"mean_tvd":0.34816829856320297,"n_tvd":50.0,"n":798.0,"n_ece_hard":277.0,"score":59.98292420575325},"noul":{"ece":0.14919191919191907,"brier":0.18700707070707095,"n":495.0,"score":70.16161616161618,"abstention_rate":0.19797979797979803},"score":{"nrps":0.10820997597418104,"top_ece":0.11682168123766032,"n":329.0,"interval90_coverage":0.9209726443768997,"interval90_width":0.36119554204660564,"score":82.90733307752492}}},"speed":{"p50_s_raw":0.1806289586238563,"p95_s_raw":0.3019020542968048,"p50_s_adjusted":0.5112579172477126,"p95_s_adjusted":0.7538041085936096,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.09682374384236453,"basis":"price floor: base-model reference price applied"},"rank":30,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":30,"B":30,"C":34}},{"key":"open-alternative-jev","display":"open-alternative-jev (Qwen3.5-4B, IkerMoel)","addendum":null,"author":"IkerMoel","repo":"https://github.com/ikermoel/open-alternative-jev","class":"jev-rebuild","licence":"Apache-2.0 (code and weights)","open":"yes","underlying":"Qwen/Qwen3.5-4B (frozen, BF16)","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX5090), offline read-only container","gpu":"RTX5090","adapter_id":"images/open-alternative-jev/spec.json:remote_inproc_v15.RemoteInprocV15Adapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":37.357784479174825,"calibration":76.86762970590799,"speed":91.29116775952656,"cost":63.27859776193043},"scores":{"A":33.5609546866917,"B":29.915576313185298,"C":23.306218532424786},"views":{"Balanced 33:33:33":31.288486249897765,"Emphasis on Accuracy 60:20:20":26.071037925172767,"Emphasis on Cost 20:20:60":32.78700402897485,"Emphasis on Speed 20:60:20":37.00238217276661,"Intelligence only":20.854670294515845},"composite_ci95":{"A":[25.75468612694063,41.663182217360564],"B":[22.57747692929389,37.672922118345966],"C":[17.885198699264325,28.932765428722615]},"jevbench_score":33.5609546866917,"intelligence":{"I_open":42.59591148247294,"I_sealed":32.11965747587671,"base":37.357784479174825,"gap":10.476254006596236,"excess":5.289626506516993,"penalty":1.0,"per_type_split":{"open|choice":{"cc":63.280143623961266,"tiers":{"easy":89.14681941513415,"standard":76.6051468676891,"judge":67.84232365145229,"hard":46.729338033685856},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":5.2909627245910436,"tiers":{"easy":70.58823529411764,"standard":24.242424242424242,"judge":9.80392156862746,"hard":-23.893805309734518},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":59.21662809886652,"tiers":{"easy":47.81087316628233,"standard":75.15891797938407,"judge":57.036466908840055,"hard":55.73204278427366},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":65.00076013866314,"tiers":{"easy":76.56249999999999,"standard":61.96513470681459,"judge":60.24734982332155,"hard":67.19319562575942},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-16.763636363636362,"tiers":{"easy":12.00000000000001,"standard":-16.000000000000004,"judge":-7.9999999999999964,"hard":-30.909090909090907},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":48.12184865260336,"tiers":{"easy":65.6992634007358,"standard":51.34907394770446,"judge":42.577076821697965,"hard":46.27246119119873},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":76.86762970590799,"parts":{"choice":{"ece_hard":0.1316919776022638,"ece_score":73.66160447954724,"mean_tvd":0.3123444214771244,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":71.21358116591739},"noul":{"ece":0.09963701989459954,"brier":0.17149596469772813,"n":495.0,"score":80.07259602108009,"abstention_rate":0.4242424242424242},"score":{"nrps":0.11664431905900818,"top_ece":0.14851072116323072,"n":330.0,"interval90_coverage":0.8909090909090909,"interval90_width":0.4803030303030303,"score":79.31671193072651}}},"speed":{"p50_s_raw":0.03633533196989447,"p95_s_raw":0.09179776395903874,"p50_s_adjusted":0.22267066393978893,"p95_s_adjusted":0.33359552791807745,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.0167512315270936,"basis":"ESTIMATE (proxy tokens, I-2): exact base-model market reference; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)"},"rank":31,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":31,"B":32,"C":35}},{"key":"nimble-9b","display":"Bespoke Nimble 9B (Bespoke Labs)","addendum":null,"author":"Bespoke Labs","repo":"https://github.com/bespokelabsai/nimble","class":"jev-rebuild","licence":"Apache-2.0 (weights); repository without a licence file as of 19 Sep","open":"yes","underlying":"bespokelabs/Bespoke-Nimble-9B (LoRA, adapter unchanged since 93ec5d6), merged into Qwen/Qwen3.5-9B@c202236 with the author's PEFT safe-merge","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/nimble-9b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":63.74597067366744,"calibration":77.16259866302579,"speed":82.95342573220694,"cost":36.75037423027614},"scores":{"A":31.8195434029124,"B":32.310865011937956,"C":31.8195434029124},"views":{"Balanced 33:33:33":29.492712053149084,"Emphasis on Accuracy 60:20:20":31.289963360792562,"Emphasis on Cost 20:20:60":24.696718592776953,"Emphasis on Speed 20:60:20":34.16501680960333,"Intelligence only":34.43786836751591},"composite_ci95":{"A":[31.188890807449496,32.274270442759345],"B":[31.4950648517445,32.969723723528006],"C":[31.176926835399797,32.274270442759345]},"jevbench_score":31.8195434029124,"intelligence":{"I_open":65.1457495138704,"I_sealed":62.34619183346448,"base":63.74597067366744,"gap":2.7995576804059183,"excess":-2.387069819673324,"penalty":1.0,"per_type_split":{"open|choice":{"cc":71.68287617013561,"tiers":{"easy":98.19113656918904,"standard":75.04548999220171,"judge":79.25311203319502,"hard":57.69682726204465},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":52.85520483334122,"tiers":{"easy":64.70588235294117,"standard":60.606060606060595,"judge":82.35294117647058,"hard":23.893805309734507},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":70.89916753813438,"tiers":{"easy":82.31172489122478,"standard":79.54763619054714,"judge":70.02436327449513,"hard":64.37789707138484},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":72.81744698393845,"tiers":{"easy":91.2109375,"standard":78.2657912610369,"judge":72.1731448763251,"hard":65.97812879708384},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":44.545454545454554,"tiers":{"easy":68.0,"standard":28.000000000000004,"judge":32.00000000000001,"hard":56.363636363636374},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":69.67567397100045,"tiers":{"easy":71.08838672560954,"standard":78.70853527327498,"judge":69.64710519169674,"hard":64.8274917156887},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":77.16259866302579,"parts":{"choice":{"ece_hard":0.12758339853798056,"ece_score":74.48332029240389,"mean_tvd":0.30830640533409903,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":71.826339879497},"noul":{"ece":0.09389240135955254,"brier":0.12388675340423497,"n":495.0,"score":81.22151972808949,"abstention_rate":0.15353535353535352},"score":{"nrps":0.08139624087095425,"top_ece":0.17490251574961402,"n":330.0,"interval90_coverage":0.8575757575757575,"interval90_width":0.16565656565656572,"score":78.43993638149088}}},"speed":{"p50_s_raw":0.198314493522048,"p95_s_raw":0.38837731960229444,"p50_s_adjusted":0.546628987044096,"p95_s_adjusted":0.9267546392045889,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.1283278325123153,"basis":"documented hosted-model estimate"},"rank":32,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":32,"B":31,"C":25}},{"key":"malkuth-2b","display":"Malkuth-2B (newfull5, Kev post-train)","addendum":null,"author":"newfull5 (dhtocks)","repo":"https://github.com/newfull5/malkuth","class":"jev-rebuild","licence":"CC-BY-NC-4.0, research use only (XNLI and RACE in the training mix)","open":"yes","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX5090), offline read-only container","gpu":"RTX5090","adapter_id":"images/malkuth-2b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":35.53647228073529,"calibration":75.18587487053827,"speed":91.80105854155826,"cost":65.5661191189412},"scores":{"A":29.896192877577803,"B":26.384634413814368,"C":20.761245053873477},"views":{"Balanced 33:33:33":27.915795106016912,"Emphasis on Accuracy 60:20:20":22.84336748921099,"Emphasis on Cost 20:20:60":29.78800020397366,"Emphasis on Speed 20:60:20":33.20153017322748,"Intelligence only":17.95076371724391},"composite_ci95":{"A":[20.96423569566723,38.978586221197304],"B":[18.086333645410093,35.04119294166098],"C":[14.558497010880016,27.068462653609238]},"jevbench_score":29.896192877577803,"intelligence":{"I_open":44.24414236221776,"I_sealed":28.611125630269996,"base":36.42763399624388,"gap":15.633016731947762,"excess":10.446389231868519,"penalty":0.9755361076813148,"per_type_split":{"open|choice":{"cc":54.828061836361115,"tiers":{"easy":89.14681941513415,"standard":67.24720561476475,"judge":67.84232365145229,"hard":30.27810419114765},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":26.009338570504628,"tiers":{"easy":76.47058823529412,"standard":24.242424242424242,"judge":62.745098039215684,"hard":-13.27433628318584},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":51.89502667978753,"tiers":{"easy":39.241589861751144,"standard":67.96974381625442,"judge":52.00314547837483,"hard":46.93993821712269},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":48.49372586969443,"tiers":{"easy":82.421875,"standard":53.81480642970343,"judge":44.34628975265018,"hard":40.46172539489673},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-5.781818181818181,"tiers":{"easy":12.00000000000001,"standard":-12.0,"judge":-7.9999999999999964,"hard":-5.454545454545457},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":43.12146920293375,"tiers":{"easy":59.144112318840605,"standard":52.48518052057095,"judge":37.35101095197979,"hard":38.7617964533539},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":75.18587487053827,"parts":{"choice":{"ece_hard":0.14623212996389884,"ece_score":70.75357400722024,"mean_tvd":0.29562866660000664,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":70.5953536736098},"noul":{"ece":0.1561735353535354,"brier":0.19303849723232322,"n":495.0,"score":68.76529292929293,"abstention_rate":0.29292929292929293},"score":{"nrps":0.13080165194696963,"top_ece":0.07262939393939392,"n":330.0,"interval90_coverage":0.9303030303030303,"interval90_width":0.561868686868687,"score":86.19697800871212}}},"speed":{"p50_s_raw":0.04066141549265012,"p95_s_raw":0.06777285743592065,"p50_s_adjusted":0.23132283098530024,"p95_s_adjusted":0.2855457148718413,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.014053873152709357,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":33,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":33,"B":35,"C":37}},{"key":"openjev-sglang","display":"openjev-sglang (Qwen3.6-35B-A3B on SGLang)","addendum":null,"author":"ekzhang","repo":"https://github.com/ekzhang/openjev-sglang","class":"jev-rebuild","licence":"no licence file in the repository as of 2026-09-19; Qwen3.6 weights keep their own terms","open":"yes","underlying":"Qwen3.6-35B-A3B","endpoint_kind":"demo","endpoint_condition":"the author's public demo endpoint","gpu":null,"adapter_id":"harness/specs/openjev-sglang.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":true,"api_exposure_note":"API measurement: the operator's endpoint received sealed item text, without answers.","status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":58.648935534041364,"calibration":82.87127167957337,"speed":78.0657837921845,"cost":35.63382653259791},"scores":{"A":29.02726618573374,"B":29.17633727531063,"C":27.734727492696795},"views":{"Balanced 33:33:33":26.30577347713018,"Emphasis on Accuracy 60:20:20":27.59626123075179,"Emphasis on Cost 20:20:60":22.26689638309311,"Emphasis on Speed 20:60:20":30.39800407427992,"Intelligence only":29.788254009511864},"composite_ci95":{"A":[28.422562526211088,29.40494742988873],"B":[28.346647426334666,29.79994598846062],"C":[24.04751115494833,29.40494742988873]},"jevbench_score":29.02726618573374,"intelligence":{"I_open":61.35777167533219,"I_sealed":55.94009939275054,"base":58.648935534041364,"gap":5.417672282581648,"excess":0.2310447825024058,"penalty":1.0,"per_type_split":{"open|choice":{"cc":80.55491244652758,"tiers":{"easy":96.38227313837805,"standard":79.7244606186639,"judge":93.7759336099585,"hard":67.09753231492361},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":36.38193491394949,"tiers":{"easy":58.823529411764696,"standard":21.212121212121215,"judge":62.745098039215684,"hard":18.584070796460182},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":67.13646766551949,"tiers":{"easy":36.18600197266465,"standard":79.38766508799597,"judge":67.58495177114636,"hard":68.4121222982748},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":74.59813773415907,"tiers":{"easy":85.3515625,"standard":74.19062712248135,"judge":68.19787985865725,"hard":76.91373025516404},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":27.200000000000003,"tiers":{"easy":52.0,"standard":15.999999999999993,"judge":36.00000000000001,"hard":19.999999999999996},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":66.02216044409255,"tiers":{"easy":76.97326685759016,"standard":68.10605355143156,"judge":61.5517431466956,"hard":65.59525026009632},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":82.87127167957337,"parts":{"choice":{"ece_hard":0.04011216296140964,"ece_score":91.97756740771807,"mean_tvd":0.3284542337752234,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":79.56607201509786},"noul":{"ece":0.08274897091682233,"brier":0.12604658057209445,"n":495.0,"score":83.45020581663553,"abstention_rate":0.2666666666666667},"score":{"nrps":0.0869064310144005,"top_ece":0.1005714124229327,"n":330.0,"interval90_coverage":0.9151515151515152,"interval90_width":0.3108585858585858,"score":85.5975372069867}}},"speed":{"p50_s_raw":0.5980852171778679,"p95_s_raw":0.6525271125137806,"p50_s_adjusted":1.1961704343557358,"p95_s_adjusted":1.3050542250275612,"n":450,"adjustment":"x2 demo-endpoint adjustment (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.13981028325123152,"basis":"ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)"},"rank":34,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":34,"B":33,"C":30}},{"key":"decider-35b-a3b","display":"decider-35b-a3b (Mapika)","addendum":null,"author":"Mapika","repo":"https://huggingface.co/Mapika/decider-35b-a3b","class":"jev-rebuild","licence":"Apache-2.0","open":"yes","underlying":"Qwen3.5-35B-A3B-Base with a trained decision readout, 34.7B parameters / 3B active","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/decider-35b-a3b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":60.45951150408739,"calibration":82.03924708540572,"speed":91.00421380212515,"cost":34.38579876859529},"scores":{"A":27.497459411252194,"B":27.710085669874296,"C":27.497459411252194},"views":{"Balanced 33:33:33":25.063646358718533,"Emphasis on Accuracy 60:20:20":26.365921213247887,"Emphasis on Cost 20:20:60":20.603709005545024,"Emphasis on Speed 20:60:20":30.090957262150802,"Intelligence only":28.59452323204778},"composite_ci95":{"A":[26.93393630242337,27.871066050370203],"B":[26.935050076468837,28.28516241072057],"C":[24.14994839287159,27.871066050370203]},"jevbench_score":27.497459411252194,"intelligence":{"I_open":64.142449771148,"I_sealed":56.77657323702679,"base":60.45951150408739,"gap":7.365876534121206,"excess":2.1792490340419635,"penalty":1.0,"per_type_split":{"open|choice":{"cc":74.29787656936696,"tiers":{"easy":96.38227313837805,"standard":87.52274499610087,"judge":85.47717842323651,"hard":53.779866823345074},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":51.34273500228732,"tiers":{"easy":70.58823529411764,"standard":42.42424242424243,"judge":80.3921568627451,"hard":29.20353982300885},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":66.7867377417897,"tiers":{"easy":25.374308755760357,"standard":80.14023851590106,"judge":66.86474442988204,"hard":70.4045895851721},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":76.47002057270414,"tiers":{"easy":88.28125,"standard":80.98256735340728,"judge":72.1731448763251,"hard":74.48359659781289},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":31.309090909090912,"tiers":{"easy":60.00000000000001,"standard":24.0,"judge":32.00000000000001,"hard":27.27272727272727},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":62.55060822928531,"tiers":{"easy":75.58623188405798,"standard":71.18681780016796,"judge":62.38618365627634,"hard":55.096915959907534},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":82.03924708540572,"parts":{"choice":{"ece_hard":0.12633935018050504,"ece_score":74.732129963899,"mean_tvd":0.2638981665500017,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":74.17115665444942},"noul":{"ece":0.0707870707070704,"brier":0.11012826391919195,"n":495.0,"score":85.84258585858592,"abstention_rate":0.2080808080808081},"score":{"nrps":0.08997396453030303,"top_ece":0.09397303030303017,"n":330.0,"interval90_coverage":0.9272727272727272,"interval90_width":0.33005050505050504,"score":86.10399874318183}}},"speed":{"p50_s_raw":0.045338188065215945,"p95_s_raw":0.08985994872637092,"p50_s_adjusted":0.24067637613043188,"p95_s_adjusted":0.32971989745274183,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.1538650708128079,"basis":"price floor: base-model reference price applied"},"rank":35,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":35,"B":34,"C":31}},{"key":"localjev-qwen3.5-4b","display":"local-jev Qwen3.5-4B","addendum":null,"author":"Amith Chandrappa (amithgc)","repo":"https://github.com/amithgc/local-jev","class":"jev-rebuild","licence":"MIT code; Apache-2.0 Qwen weights","open":"yes","underlying":"Qwen/Qwen3.5-4B text model, zero-shot next-token letter probabilities","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/localjev-qwen3.5-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":33.74767380040539,"calibration":82.44516243154092,"speed":83.73771259157968,"cost":59.269627658320445},"scores":{"A":25.82001585149664,"B":22.731120666929975,"C":17.93056656353933},"views":{"Balanced 33:33:33":23.38384064773755,"Emphasis on Accuracy 60:20:20":19.351186049964124,"Emphasis on Cost 20:20:60":24.707823779766677,"Emphasis on Speed 20:60:20":27.666883561690653,"Intelligence only":15.374164345077862},"composite_ci95":{"A":[19.886176149800825,32.96453052800093],"B":[17.19694397415245,29.530260078234154],"C":[13.809844548472798,22.892035088889532]},"jevbench_score":25.82001585149664,"intelligence":{"I_open":36.702724854533855,"I_sealed":30.79262274627692,"base":33.74767380040539,"gap":5.910102108256936,"excess":0.7234746081776935,"penalty":1.0,"per_type_split":{"open|choice":{"cc":73.6696761367919,"tiers":{"easy":96.38227313837805,"standard":79.7244606186639,"judge":89.6265560165975,"hard":52.996474735605155},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-23.237896928682975,"tiers":{"easy":41.176470588235304,"standard":0.0,"judge":-21.568627450980394,"hard":-52.212389380530965},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":59.67639535549263,"tiers":{"easy":47.47615207373271,"standard":75.60719964664311,"judge":59.13073394495414,"hard":55.170300088261236},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":69.06095945143655,"tiers":{"easy":76.56249999999999,"standard":70.11546298392577,"judge":60.24734982332155,"hard":73.2685297691373},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-38.290909090909096,"tiers":{"easy":-19.999999999999996,"standard":-40.0,"judge":-24.0,"hard":-52.727272727272734},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":61.60781787830332,"tiers":{"easy":73.30018115942029,"standard":64.31759026028547,"judge":61.74161752316767,"hard":57.22949113338475},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":82.44516243154092,"parts":{"choice":{"ece_hard":0.08462599277978346,"ece_score":83.07480144404332,"mean_tvd":0.230396499916665,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":80.01757572618841},"noul":{"ece":0.09167757575757576,"brier":0.13444001785858598,"n":495.0,"score":81.66448484848485,"abstention_rate":0.6161616161616161},"score":{"nrps":0.10311631408585854,"top_ece":0.0919075757575757,"n":330.0,"interval90_coverage":0.9454545454545454,"interval90_width":0.5088383838383839,"score":85.65342671994951}}},"speed":{"p50_s_raw":0.138433791231364,"p95_s_raw":0.42034247601404706,"p50_s_adjusted":0.42686758246272805,"p95_s_adjusted":0.9906849520280941,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.022786576354679802,"basis":"documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)"},"rank":36,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":36,"B":38,"C":43}},{"key":"open-jev-zefan-9b","display":"Open-Jev 9B (Zefan Cai)","addendum":null,"author":"Zefan Cai (@Zefan_Cai)","repo":"https://github.com/Zefan-Cai/Open-Jev","class":"jev-rebuild","licence":"MIT (loader); Apache-2.0 (adapter and pinned Qwen base); CC0-1.0 public training projection","open":"yes","underlying":"Qwen3.5-9B plus rank-8 LoRA and trained scalar decision head","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/open-jev-zefan-9b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":63.82052863874054,"calibration":81.52933105829675,"speed":73.58523755072761,"cost":33.05490478272432},"scores":{"A":24.35608345131106,"B":24.98980978448973,"C":24.35608345131106},"views":{"Balanced 33:33:33":22.031978386347873,"Emphasis on Accuracy 60:20:20":24.053635358614148,"Emphasis on Cost 20:20:60":18.207909890804228,"Emphasis on Speed 20:60:20":25.20747778317762,"Intelligence only":27.89280621034464},"composite_ci95":{"A":[23.927044930797685,24.64472014937102],"B":[24.396050570509544,25.44585523866238],"C":[23.926502881115063,24.64472014937102]},"jevbench_score":24.35608345131106,"intelligence":{"I_open":61.78593118508326,"I_sealed":65.85512609239782,"base":63.82052863874054,"gap":-4.069194907314568,"excess":-9.255822407393811,"penalty":1.0,"per_type_split":{"open|choice":{"cc":66.19142369957882,"tiers":{"easy":96.38227313837805,"standard":73.48583311671433,"judge":71.99170124481329,"hard":50.64629847238542},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":48.95919107787926,"tiers":{"easy":76.47058823529412,"standard":57.57575757575757,"judge":62.745098039215684,"hard":27.43362831858407},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":70.20717877779171,"tiers":{"easy":58.95041897615663,"standard":81.3647374208426,"judge":69.73772163647239,"hard":67.79468226266454},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":71.45157070104113,"tiers":{"easy":88.28125,"standard":75.5490151686665,"judge":65.54770318021201,"hard":69.62332928311056},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":58.690909090909095,"tiers":{"easy":84.00000000000001,"standard":43.99999999999999,"judge":68.0,"hard":52.727272727272734},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":67.42289848524325,"tiers":{"easy":79.5403850699516,"standard":72.65294815521406,"judge":69.55506458520617,"hard":60.17937742910857},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":81.52933105829675,"parts":{"choice":{"ece_hard":0.1106033375245877,"ece_score":77.87933249508245,"mean_tvd":0.28171887066993206,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":74.85372271404462},"noul":{"ece":0.09714938165037393,"brier":0.12342402792417258,"n":495.0,"score":80.57012366992522,"abstention_rate":0.13131313131313127},"score":{"nrps":0.0776873899926604,"top_ece":0.06951483709446572,"n":330.0,"interval90_coverage":0.9303030303030303,"interval90_width":0.28939393939393937,"score":89.1641467909204}}},"speed":{"p50_s_raw":0.5631460719741881,"p95_s_raw":1.640916511422256,"p50_s_adjusted":1.276292143948376,"p95_s_adjusted":3.4318330228445117,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.17041317733990147,"basis":"documented hosted-model estimate"},"rank":37,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":37,"B":36,"C":33}},{"key":"decision-2b","display":"Decision 2B (FlyMy.AI, v59)","addendum":null,"author":"FlyMy.AI (@denti)","repo":"https://huggingface.co/flymy-ai/decision-2b-preview","class":"jev-rebuild","licence":"Apache-2.0 notices on the included code and the pinned base; the weights are an evaluation preview under EVALUATION-PERMISSION.md, not a cleared commercial release","open":"yes","underlying":"openbmb/MiniCPM5-2B with a trained LoRA adapter and pointer head (26.2M trainable parameters)","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX6000), offline read-only container","gpu":"RTX6000","adapter_id":"images/decision-2b/spec.json:flymy_v15.FlyMyPackage","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":31.312216031505486,"calibration":86.21585443681741,"speed":90.35616211568993,"cost":66.37275756963365},"scores":{"A":22.51688642767283,"B":19.299273543136604,"C":15.636726685883911},"views":{"Balanced 33:33:33":20.26076694157444,"Emphasis on Accuracy 60:20:20":16.080545286324668,"Emphasis on Cost 20:20:60":22.231781041422483,"Emphasis on Speed 20:60:20":24.44878868072661,"Intelligence only":12.280085914548287},"composite_ci95":{"A":[16.675348522547402,28.773372628345562],"B":[14.076770422866533,25.124356365137665],"C":[11.580103140657918,19.98150876968442]},"jevbench_score":22.51688642767283,"intelligence":{"I_open":34.645695446630285,"I_sealed":27.978736616380687,"base":31.312216031505486,"gap":6.666958830249598,"excess":1.4803313301703556,"penalty":1.0,"per_type_split":{"open|choice":{"cc":58.265190419661025,"tiers":{"easy":98.19113656918904,"standard":67.24720561476475,"judge":55.39419087136931,"hard":45.94594594594594},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-16.10777215149938,"tiers":{"easy":58.823529411764696,"standard":12.12121212121211,"judge":-11.764705882352944,"hard":-52.212389380530965},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":61.77966807172922,"tiers":{"easy":33.683871034295954,"standard":71.51419167650035,"judge":69.32108617250525,"hard":58.280291953119935},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":67.70410134214103,"tiers":{"easy":85.3515625,"standard":64.68191079918498,"judge":62.897526501766784,"hard":68.408262454435},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-35.127272727272725,"tiers":{"easy":12.00000000000001,"standard":-31.999999999999996,"judge":-43.99999999999999,"hard":-41.81818181818182},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":51.359381234273755,"tiers":{"easy":62.88746852162508,"standard":58.88458396911487,"judge":45.314711352785054,"hard":49.24826045613189},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":86.21585443681741,"parts":{"choice":{"ece_hard":0.052297166829797445,"ece_score":89.5405666340405,"mean_tvd":0.2250098243022926,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":83.51979210190561},"noul":{"ece":0.06815406545421997,"brier":0.15106906880962623,"n":495.0,"score":86.369186909156,"abstention_rate":0.5797979797979798},"score":{"nrps":0.10797650023219298,"top_ece":0.05842590688999745,"n":330.0,"interval90_coverage":0.9545454545454546,"interval90_width":0.5267676767676768,"score":88.7585842993906}}},"speed":{"p50_s_raw":0.07489005557727069,"p95_s_raw":0.07865651411702856,"p50_s_adjusted":0.2997801111545414,"p95_s_adjusted":0.30731302823405715,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.013210160098522166,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":38,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":38,"B":41,"C":45}},{"key":"gpt-5.6-luna","display":"GPT-5.6 Luna (low reasoning effort)","addendum":null,"author":"OpenAI","repo":null,"class":"llm-baseline","licence":"proprietary API","open":"no","underlying":"closed","endpoint_kind":"api","endpoint_condition":"the operator's hosted API","gpu":null,"adapter_id":"harness/specs/gpt-5.6-luna.json:jevbench.adapters.openai_compat.OpenAICompatAdapter","api_flag":true,"api_exposure_note":"API measurement: the operator's endpoint received sealed item text, without answers.","status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":94.33090883309077,"calibration":94.71076759662373,"speed":74.06258245488996,"cost":30.66824417593942},"scores":{"A":22.371016433424646,"B":24.156847114641927,"C":22.371016433424646},"views":{"Balanced 33:33:33":19.902091456402744,"Emphasis on Accuracy 60:20:20":24.143664413958902,"Emphasis on Cost 20:20:60":15.42830299146938,"Emphasis on Speed 20:60:20":22.470268722064183,"Intelligence only":35.48884250789443},"composite_ci95":{"A":[22.260921433578396,22.46343987903718],"B":[24.004102294684866,24.28792813200078],"C":[22.260921433578396,22.46343987903718]},"jevbench_score":22.371016433424646,"intelligence":{"I_open":93.15592763913932,"I_sealed":95.50589002704224,"base":94.33090883309077,"gap":-2.34996238790292,"excess":-7.536589887982163,"penalty":1.0,"per_type_split":{"open|choice":{"cc":95.9370980705296,"tiers":{"easy":100.0,"standard":95.32102937353783,"judge":96.88796680497926,"hard":94.51625538582061},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":87.11166848074708,"tiers":{"easy":100.0,"standard":93.93939393939394,"judge":94.11764705882352,"hard":75.2212389380531},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":96.41901636614128,"tiers":{"easy":92.19354838709677,"standard":99.09187279151944,"judge":95.78636959370904,"hard":96.61344022753751},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":93.85225740986291,"tiers":{"easy":94.140625,"standard":91.84967172288883,"judge":93.37455830388691,"hard":95.13973268529769},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":94.14545454545454,"tiers":{"easy":100.0,"standard":96.0,"judge":87.99999999999999,"hard":96.36363636363636},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":98.5199581258093,"tiers":{"easy":99.64673913043478,"standard":99.45340050377834,"judge":99.59519797809604,"hard":96.96511179645336},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":94.71076759662373,"parts":{"choice":{"ece_hard":0.059550371644044475,"ece_score":88.08992567119111,"mean_tvd":0.04208186321201166,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":91.94086967499497},"noul":{"ece":0.030852003673131213,"brier":0.034674078902818234,"n":495.0,"score":93.82959926537376,"abstention_rate":0.016161616161616155},"score":{"nrps":0.005979484626312506,"top_ece":0.013391919191818978,"n":330.0,"interval90_coverage":0.9787878787878788,"interval90_width":0.006818181818181818,"score":98.36183384950247}}},"speed":{"p50_s_raw":1.3211710378527641,"p95_s_raw":2.9701795924454926,"p50_s_adjusted":1.3211710378527641,"p95_s_adjusted":2.9701795924454926,"n":450,"adjustment":"none (hosted API, measured as is)"},"cost":{"kind":"tariff","usd_per_1000":0.20467204433497538,"basis":"operator list price"},"rank":39,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":39,"B":37,"C":36}},{"key":"typecastlm","display":"typecastlm (Mikhail Gribov, Qwen3.5-4B computed head)","addendum":null,"author":"Mikhail Gribov","repo":"https://github.com/mihail-gribov/typecastlm","class":"system-one-open","licence":"Apache-2.0 (package and weights)","open":"yes","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX5090), offline read-only container","gpu":"RTX5090","adapter_id":"images/typecastlm/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":31.24114221399841,"calibration":76.71047738548705,"speed":92.0380396136627,"cost":63.95900966831593},"scores":{"A":21.826965316555995,"B":18.850194162174848,"C":15.157614803163888},"views":{"Balanced 33:33:33":20.017562533976484,"Emphasis on Accuracy 60:20:20":15.93128916525674,"Emphasis on Cost 20:20:60":21.742432680793506,"Emphasis on Speed 20:60:20":24.32745311425757,"Intelligence only":12.196653974015371},"composite_ci95":{"A":[16.231764971521944,28.575247201608622],"B":[13.768179560326216,25.18177166866868],"C":[11.272059008001348,19.84392166778376]},"jevbench_score":21.826965316555995,"intelligence":{"I_open":23.699002495040826,"I_sealed":38.78328193295599,"base":31.24114221399841,"gap":-15.08427943791516,"excess":-20.270906937994404,"penalty":1.0,"per_type_split":{"open|choice":{"cc":66.45284701040082,"tiers":{"easy":92.7645462767561,"standard":84.40343124512606,"judge":68.87966804979253,"hard":49.0795142969056},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-52.089347404287544,"tiers":{"easy":-11.764705882352944,"standard":-45.45454545454546,"judge":-74.50980392156863,"hard":-48.67256637168141},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":56.73350787900921,"tiers":{"easy":32.6273750480858,"standard":64.69949789703816,"judge":60.03354979086002,"hard":56.30201464383748},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":72.00706113946435,"tiers":{"easy":82.421875,"standard":74.19062712248135,"judge":73.49823321554771,"hard":67.19319562575942},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-7.490909090909094,"tiers":{"easy":12.00000000000001,"standard":-12.0,"judge":-4.0000000000000036,"hard":-12.727272727272732},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":51.83369375031271,"tiers":{"easy":68.33244307522665,"standard":53.5615139862305,"judge":48.685030196219714,"hard":49.20659396669508},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":76.71047738548705,"parts":{"choice":{"ece_hard":0.08098920458616882,"ece_score":83.80215908276624,"mean_tvd":0.3165516236083484,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":76.0734983609657},"noul":{"ece":0.13493836400492118,"brier":0.21792086351560655,"n":495.0,"score":73.01232719901576,"abstention_rate":0.5858585858585859},"score":{"nrps":0.10993623801474207,"top_ece":0.13457581502783178,"n":330.0,"interval90_coverage":0.9757575757575757,"interval90_width":0.8330808080808081,"score":81.04560659647973}}},"speed":{"p50_s_raw":0.03754384949570522,"p95_s_raw":0.06393583689932711,"p50_s_adjusted":0.22508769899141043,"p95_s_adjusted":0.2778716737986542,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.01589887315270936,"basis":"documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)"},"rank":40,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":40,"B":45,"C":47}},{"key":"jev-qwen3.5-9b-base-nvfp4","display":"JEV Qwen3.5-9B Base NVFP4","addendum":null,"author":"WilfLin","repo":"https://huggingface.co/WIlfLin/JEV-Qwen3.5-9B-Base-NVFP4","class":"jev-rebuild","licence":"Apache-2.0 entrant and upstream checkpoint","open":"yes","underlying":"Qwen3.5-9B Base NVFP4 with compact BF16 decision head","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX5090), offline read-only container","gpu":"RTX5090","adapter_id":"images/jev-qwen3.5-9b-base-nvfp4/spec.json:jev_choice_http.Adapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":32.24194556070419,"calibration":80.93292191627495,"speed":93.53733828267468,"cost":47.592235403520306},"scores":{"A":20.07262599814848,"B":17.755447982336538,"C":13.939323609825337},"views":{"Balanced 33:33:33":18.020299995518467,"Emphasis on Accuracy 60:20:20":15.099645707236771,"Emphasis on Cost 20:20:60":17.983913005615214,"Emphasis on Speed 20:60:20":22.39794913832311,"Intelligence only":12.146633055154606},"composite_ci95":{"A":[15.1940766297855,25.505644598215827],"B":[13.195795378879625,22.94062856057887],"C":[10.551442104017708,17.712253193205427]},"jevbench_score":20.07262599814848,"intelligence":{"I_open":32.876088752154395,"I_sealed":31.60780236925398,"base":32.24194556070419,"gap":1.2682863829004134,"excess":-3.918341117178829,"penalty":1.0,"per_type_split":{"open|choice":{"cc":71.16709783601964,"tiers":{"easy":96.38227313837805,"standard":75.04548999220171,"judge":89.6265560165975,"hard":49.0795142969056},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-34.85432145505024,"tiers":{"easy":5.882352941176472,"standard":-39.39393939393939,"judge":-52.94117647058824,"hard":-29.20353982300885},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":62.3154898754938,"tiers":{"easy":31.71997571338293,"standard":73.94858718392696,"judge":57.44464977819541,"hard":67.80094983477873},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":59.86656059671132,"tiers":{"easy":79.4921875,"standard":63.32352275299977,"judge":50.97173144876326,"hard":59.90279465370596},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-22.472727272727273,"tiers":{"easy":12.00000000000001,"standard":-12.0,"judge":-19.999999999999996,"hard":-38.18181818181819},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":57.42957378377791,"tiers":{"easy":72.8641754647496,"standard":62.32447323157109,"judge":53.13911824245885,"hard":54.34131529562769},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":80.93292191627495,"parts":{"choice":{"ece_hard":0.07197345707175538,"ece_score":85.60530858564891,"mean_tvd":0.24738028711851867,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":80.43363993689852},"noul":{"ece":0.13810909353384762,"brier":0.1516578693794212,"n":495.0,"score":72.37818129323048,"abstention_rate":0.6161616161616161},"score":{"nrps":0.09632665434693449,"top_ece":0.05196722763957396,"n":330.0,"interval90_coverage":0.9424242424242424,"interval90_width":0.49343434343434345,"score":89.98694451869588}}},"speed":{"p50_s_raw":0.019789331941865385,"p95_s_raw":0.04180104220286005,"p50_s_adjusted":0.18957866388373076,"p95_s_adjusted":0.2336020844057201,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.05583743842364531,"basis":"ESTIMATE (proxy tokens, I-2): exact base-model market reference"},"rank":41,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":41,"B":47,"C":48}},{"key":"gemini-3.1-flash-lite","display":"Gemini 3.1 Flash-Lite","addendum":null,"author":"Google","repo":null,"class":"llm-baseline","licence":"proprietary API","open":"no","underlying":"closed","endpoint_kind":"api","endpoint_condition":"the operator's hosted API","gpu":null,"adapter_id":"harness/specs/gemini-3.1-flash-lite.json:jevbench.adapters.openai_compat.OpenAICompatAdapter","api_flag":true,"api_exposure_note":"API measurement: the operator's endpoint received sealed item text, without answers.","status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":77.58771823015074,"calibration":74.67587927109267,"speed":80.05303137744917,"cost":29.76239303895973},"scores":{"A":19.583532784444895,"B":20.778880829756616,"C":19.583532784444895},"views":{"Balanced 33:33:33":18.022440572612815,"Emphasis on Accuracy 60:20:20":20.90208919224148,"Emphasis on Cost 20:20:60":14.040404841077358,"Emphasis on Speed 20:60:20":21.099709608322517,"Intelligence only":27.490881546260717},"composite_ci95":{"A":[19.303624954036483,19.83181713878822],"B":[20.410436729614165,21.113109439966298],"C":[19.303624954036483,19.83181713878822]},"jevbench_score":19.583532784444895,"intelligence":{"I_open":76.61549398603603,"I_sealed":78.55994247426545,"base":77.58771823015074,"gap":-1.9444484882294262,"excess":-7.131075988308669,"penalty":1.0,"per_type_split":{"open|choice":{"cc":83.98327600141056,"tiers":{"easy":96.38227313837805,"standard":95.32102937353783,"judge":95.850622406639,"hard":66.31414022718371},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":73.60986228763429,"tiers":{"easy":88.23529411764706,"standard":81.81818181818181,"judge":82.35294117647058,"hard":59.29203539823009},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":72.25334366906326,"tiers":{"easy":65.81797235023042,"standard":97.773851590106,"judge":69.40366972477065,"hard":63.239187996469546},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":84.09321219590066,"tiers":{"easy":94.140625,"standard":83.69934344577767,"judge":84.09893992932862,"hard":81.77399756986635},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":75.05454545454546,"tiers":{"easy":92.0,"standard":76.0,"judge":84.00000000000001,"hard":63.63636363636365},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":76.53206977235027,"tiers":{"easy":75.89673913043478,"standard":76.44836272040303,"judge":72.95703454085931,"hard":79.41403238242097},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":74.67587927109267,"parts":{"choice":{"ece_hard":0.2320206406509653,"ece_score":53.59587186980694,"mean_tvd":0.0602453253801704,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":73.78566966589494},"noul":{"ece":0.10700429752066044,"brier":0.11326300106119025,"n":495.0,"score":78.59914049586791,"abstention_rate":0.008080808080808133},"score":{"nrps":0.08817375000000002,"top_ece":0.23948484848484825,"n":330.0,"interval90_coverage":0.7606060606060606,"interval90_width":0.015404040404040406,"score":71.64282765151518}}},"speed":{"p50_s_raw":0.8633867427706718,"p95_s_raw":1.1441724501550194,"p50_s_adjusted":0.8633867427706718,"p95_s_adjusted":1.1441724501550194,"n":450,"adjustment":"none (hosted API, measured as is)"},"cost":{"kind":"tariff","usd_per_1000":0.21940855911330048,"basis":"operator list price"},"rank":42,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":42,"B":39,"C":38}},{"key":"ninfer-qwen3.8-27b","display":"NInfer Qwen3.8-27B NVFP4","addendum":null,"author":"Igor L. / NInfer contributors","repo":"https://github.com/igorls/ninfer","class":"native-logit","licence":"Apache-2.0 (engine and submitted 27B artifact/base)","open":"yes","underlying":"Qwen/Qwen3.8-27B, NInfer NVFP4/FP8 artifact","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX5090), offline read-only container","gpu":"RTX5090","adapter_id":"images/ninfer-qwen3.8-27b/spec.json:ninfer_native.NInferNativeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":65.47395819302696,"calibration":85.91195579796471,"speed":89.8610845330764,"cost":29.118887844204664},"scores":{"A":18.742649798238652,"B":19.34617225634836,"C":18.742649798238652},"views":{"Balanced 33:33:33":16.750622611223033,"Emphasis on Accuracy 60:20:20":18.576177137127328,"Emphasis on Cost 20:20:60":13.102469983124392,"Emphasis on Speed 20:60:20":20.431537156152054,"Intelligence only":22.20639984766575},"composite_ci95":{"A":[18.45104757628145,18.922589561307248],"B":[18.92985911551215,19.64828120753878],"C":[18.45104757628145,18.922589561307248]},"jevbench_score":18.742649798238652,"intelligence":{"I_open":66.97566042468935,"I_sealed":63.97225596136457,"base":65.47395819302696,"gap":3.0034044633247845,"excess":-2.183223036754458,"penalty":1.0,"per_type_split":{"open|choice":{"cc":80.86187003722651,"tiers":{"easy":96.38227313837805,"standard":89.08240187158823,"judge":92.73858921161826,"hard":63.963963963963955},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":47.772309245500296,"tiers":{"easy":82.35294117647058,"standard":57.57575757575757,"judge":68.62745098039215,"hard":18.584070796460182},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":72.29280199134126,"tiers":{"easy":56.96217831131241,"standard":83.55170403037255,"judge":77.20210812013953,"hard":66.81402729523414},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":80.00292218906678,"tiers":{"easy":85.3515625,"standard":86.41611953814805,"judge":74.82332155477033,"hard":79.34386391251519},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":44.727272727272734,"tiers":{"easy":76.0,"standard":24.0,"judge":52.0,"hard":41.81818181818182},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":67.18657296775422,"tiers":{"easy":76.74621676344742,"standard":69.58612480739778,"judge":63.90748423596986,"hard":66.05620264784741},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":85.91195579796471,"parts":{"choice":{"ece_hard":0.07320507155126452,"ece_score":85.35898568974709,"mean_tvd":0.29594025310350514,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":77.88248018969828},"noul":{"ece":0.061787820545642955,"brier":0.09997090924891203,"n":495.0,"score":87.64243589087141,"abstention_rate":0.20404040404040402},"score":{"nrps":0.07114015360017155,"top_ece":0.042320410066669624,"n":330.0,"interval90_coverage":0.9606060606060606,"interval90_width":0.3474747474747474,"score":92.21095131332446}}},"speed":{"p50_s_raw":0.0491448815446347,"p95_s_raw":0.13292309776879818,"p50_s_adjusted":0.2482897630892694,"p95_s_adjusted":0.4158461955375964,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.2305174137931034,"basis":"price floor: base-model reference price applied"},"rank":43,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":43,"B":40,"C":39}},{"key":"ninfer-qwen3.8-27b-t1.5","display":"NInfer Qwen3.8-27B NVFP4 (T=1.5)","addendum":null,"author":"Igor L. / NInfer contributors","repo":"https://github.com/igorls/ninfer","class":"native-logit","licence":"Apache-2.0 (engine and submitted 27B artifact/base)","open":"yes","underlying":"Qwen/Qwen3.8-27B, NInfer NVFP4/FP8 artifact","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX5090), offline read-only container","gpu":"RTX5090","adapter_id":"images/ninfer-qwen3.8-27b-t1.5/derive_from_27b.py:derive_t15","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":61.07759382196298,"calibration":86.49259657807198,"speed":89.8610845330764,"cost":29.118887844204664},"scores":{"A":18.48189600644971,"B":18.88920290519583,"C":18.48189600644971},"views":{"Balanced 33:33:33":16.452850206736613,"Emphasis on Accuracy 60:20:20":17.928461730599007,"Emphasis on Cost 20:20:60":12.99211382052128,"Emphasis on Speed 20:60:20":20.164451202362645,"Intelligence only":20.715311974040358},"composite_ci95":{"A":[18.17854646267274,18.648169861543884],"B":[18.444739178298107,19.20540289130532],"C":[16.867965556382803,18.648169861543884]},"jevbench_score":18.48189600644971,"intelligence":{"I_open":62.432554991885866,"I_sealed":59.72263265204009,"base":61.07759382196298,"gap":2.7099223398457752,"excess":-2.4767051602334673,"penalty":1.0,"per_type_split":{"open|choice":{"cc":80.86187003722651,"tiers":{"easy":96.38227313837805,"standard":89.08240187158823,"judge":92.73858921161826,"hard":63.963963963963955},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":36.82725222027669,"tiers":{"easy":76.47058823529412,"standard":39.393939393939405,"judge":62.745098039215684,"hard":6.194690265486735},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":69.60854271815441,"tiers":{"easy":53.10192902290834,"standard":79.39270470091643,"judge":74.3533274635788,"hard":65.28452659151662},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":80.00292218906678,"tiers":{"easy":85.3515625,"standard":86.41611953814805,"judge":74.82332155477033,"hard":79.34386391251519},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":36.36363636363636,"tiers":{"easy":68.0,"standard":8.000000000000007,"judge":52.0,"hard":30.909090909090907},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":62.801339403417145,"tiers":{"easy":74.35069080100433,"standard":66.19026700915842,"judge":59.795755472073495,"hard":60.47372569965746},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":86.49259657807198,"parts":{"choice":{"ece_hard":0.046565725487101334,"ece_score":90.68685490257973,"mean_tvd":0.2652805775311959,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":82.07939857473006},"noul":{"ece":0.050195049032060304,"brier":0.09459883054585931,"n":495.0,"score":89.96099019358795,"abstention_rate":0.2828282828282829},"score":{"nrps":0.07411353495537272,"top_ece":0.08856922286333445,"n":330.0,"interval90_coverage":0.9848484848484849,"interval90_width":0.4818181818181818,"score":87.43740096589792}}},"speed":{"p50_s_raw":0.0491448815446347,"p95_s_raw":0.13292309776879818,"p50_s_adjusted":0.2482897630892694,"p95_s_adjusted":0.4158461955375964,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.2305174137931034,"basis":"price floor: base-model reference price applied"},"rank":44,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":44,"B":43,"C":40}},{"key":"instinct","display":"Instinct (ZooWork, Qwen3.8-27B)","addendum":null,"author":"rayrain-srp (ZooWork)","repo":"https://github.com/fstandhartinger/jevbench/issues/69","class":"jev-rebuild","licence":"proprietary service over Apache-2.0 base weights (Qwen3.8-27B)","open":"no","underlying":"Qwen3.8-27B (Apache-2.0 base weights) with no additional fine-tuning; their own serving stack reads candidate-token scores in one forward pass per question and normalizes them into a decision probability, with no autoregressive decoding. The serving implementation is not public.","endpoint_kind":"demo","endpoint_condition":"the author's public demo endpoint","gpu":null,"adapter_id":"harness/specs/instinct.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":true,"api_exposure_note":"API measurement: the operator's endpoint received sealed item text, without answers.","status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":62.74758857686837,"calibration":84.95658090858146,"speed":81.68029758765765,"cost":29.16007113367894},"scores":{"A":18.324676123308297,"B":18.85789359077411,"C":18.324676123308297},"views":{"Balanced 33:33:33":16.33295307078797,"Emphasis on Accuracy 60:20:20":18.02517125337854,"Emphasis on Cost 20:20:60":12.975872256759336,"Emphasis on Speed 20:60:20":19.55658931728601,"Intelligence only":21.341954505243862},"composite_ci95":{"A":[18.03251754045729,18.509314642890757],"B":[18.423335678285298,19.174218341638475],"C":[17.615351400843892,18.509314642890757]},"jevbench_score":18.324676123308297,"intelligence":{"I_open":61.912899747276896,"I_sealed":63.582277406459845,"base":62.74758857686837,"gap":-1.669377659182949,"excess":-6.856005159262192,"penalty":1.0,"per_type_split":{"open|choice":{"cc":84.48610259077087,"tiers":{"easy":98.19113656918904,"standard":90.64205874707565,"judge":93.7759336099585,"hard":71.01449275362319},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":28.500465351063998,"tiers":{"easy":70.58823529411764,"standard":42.42424242424243,"judge":25.49019607843137,"hard":13.27433628318584},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":72.75213129999584,"tiers":{"easy":42.05506928611551,"standard":82.48342931697461,"judge":74.99860827508381,"hard":73.87589006366055},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":79.66534680747468,"tiers":{"easy":91.2109375,"standard":78.2657912610369,"judge":78.79858657243817,"hard":78.12879708383961},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":39.56363636363636,"tiers":{"easy":68.0,"standard":36.00000000000001,"judge":43.99999999999999,"hard":30.909090909090907},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":71.51784904826852,"tiers":{"easy":78.33556811423234,"standard":72.1281289243048,"judge":68.0999921739055,"hard":72.07167199953165},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":84.95658090858146,"parts":{"choice":{"ece_hard":0.041775188617254565,"ece_score":91.6449622765491,"mean_tvd":0.27112430615970573,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":82.26626583028926},"noul":{"ece":0.09012706875176714,"brier":0.08283135625470438,"n":495.0,"score":81.97458624964658,"abstention_rate":0.3171717171717172},"score":{"nrps":0.06592061964396308,"top_ece":0.06075078371993284,"n":330.0,"interval90_coverage":0.9636363636363636,"interval90_width":0.3207070707070707,"score":90.62889064580857}}},"speed":{"p50_s_raw":0.259529784321785,"p95_s_raw":0.6542188361287111,"p50_s_adjusted":0.51905956864357,"p95_s_adjusted":1.3084376722574222,"n":450,"adjustment":"x2 demo-endpoint adjustment (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.22978991379310343,"basis":"ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)"},"rank":45,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":45,"B":44,"C":41}},{"key":"openjev-thinking","display":"OpenJev (thinking, BF16)","addendum":null,"author":"razorback16","repo":"https://github.com/razorback16/openjev","class":"jev-rebuild","licence":"Apache-2.0","open":"yes","underlying":"google/diffusiongemma-26b-a4b-it, BF16; OpenJev think=512","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/openjev-thinking/spec.json:openjev_thinking.ThinkingOpenJev","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":84.20554946501295,"calibration":83.10226165901825,"speed":73.86286987310874,"cost":28.51480924396573},"scores":{"A":17.939903136855065,"B":19.269266611296747,"C":17.939903136855065},"views":{"Balanced 33:33:33":16.131810682141285,"Emphasis on Accuracy 60:20:20":19.305333345098983,"Emphasis on Cost 20:20:60":12.449501761513527,"Emphasis on Speed 20:60:20":18.57206532391989,"Intelligence only":27.386822476131922},"composite_ci95":{"A":[17.75987145107232,18.078147584373728],"B":[19.028507279349867,19.473239838287018],"C":[17.75987145107232,18.078147584373728]},"jevbench_score":17.939903136855065,"intelligence":{"I_open":82.51224259391168,"I_sealed":85.89885633611422,"base":84.20554946501295,"gap":-3.3866137422025417,"excess":-8.573241242281785,"penalty":1.0,"per_type_split":{"open|choice":{"cc":81.66804459050107,"tiers":{"easy":98.19113656918904,"standard":90.64205874707565,"judge":92.73858921161826,"hard":64.74735605170387},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":80.61284368936633,"tiers":{"easy":100.0,"standard":90.90909090909092,"judge":86.27450980392157,"hard":66.3716814159292},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":85.25583950186767,"tiers":{"easy":96.95557988178955,"standard":94.28149345965278,"judge":83.79455455177514,"hard":78.91404114056402},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":78.51164590189032,"tiers":{"easy":100.0,"standard":78.2657912610369,"judge":80.12367491166077,"hard":72.05346294046174},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":88.61818181818181,"tiers":{"easy":100.0,"standard":100.0,"judge":96.0,"hard":74.54545454545453},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":90.56674128827055,"tiers":{"easy":99.95508182759957,"standard":85.36939599599363,"judge":94.12127816756136,"hard":88.15242614010864},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":83.10226165901825,"parts":{"choice":{"ece_hard":0.14617801394299762,"ece_score":70.76439721140048,"mean_tvd":0.34613564131126073,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":68.07541654013721},"noul":{"ece":0.046908044598036464,"brier":0.048015879012664964,"n":495.0,"score":90.61839108039271,"abstention_rate":0.036363636363636376},"score":{"nrps":0.03461684887491551,"top_ece":0.07656180199729376,"n":330.0,"interval90_coverage":0.9272727272727272,"interval90_width":0.04924242424242424,"score":90.61297735652485}}},"speed":{"p50_s_raw":0.7056447945069522,"p95_s_raw":1.2408293015789242,"p50_s_adjusted":1.5612895890139042,"p95_s_adjusted":2.6316586031578484,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.24145693965517237,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":46,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":46,"B":42,"C":42}},{"key":"djev-thinking","display":"djev (thinking)","addendum":null,"author":"David Villalon / Maisa","repo":"https://github.com/Davipar/djev-dev","class":"jev-rebuild","licence":"Apache-2.0","open":"yes","underlying":"google/diffusiongemma-26b-a4b-it, BF16; full generation with thinking enabled","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/djev-thinking/spec.json:djev_thinking_v15.DjevThinkingV15Adapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1357},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":77.34831993443525,"calibration":95.66103904664365,"speed":72.31810288828999,"cost":28.131942787244554},"scores":{"A":17.403898604323025,"B":18.472410565292908,"C":17.403898604323025},"views":{"Balanced 33:33:33":15.243019264957107,"Emphasis on Accuracy 60:20:20":17.95382784931123,"Emphasis on Cost 20:20:60":11.86545691772876,"Emphasis on Speed 20:60:20":17.59487888695899,"Intelligence only":24.485576136504655},"composite_ci95":{"A":[17.276321176749658,17.507045027661402],"B":[18.270980597540277,18.646681574730966],"C":[17.276321176749658,17.507045027661402]},"jevbench_score":17.403898604323025,"intelligence":{"I_open":76.55371800448589,"I_sealed":78.14292186438462,"base":77.34831993443525,"gap":-1.5892038598987313,"excess":-6.775831359977974,"penalty":1.0,"per_type_split":{"open|choice":{"cc":64.76721084087255,"tiers":{"easy":94.57340970756708,"standard":81.28411749415127,"judge":63.6929460580913,"hard":49.8629063846455},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":74.62858675248056,"tiers":{"easy":100.0,"standard":93.93939393939394,"judge":66.66666666666667,"hard":64.60176991150442},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":90.26535642010458,"tiers":{"easy":61.73963133640552,"standard":95.26956582584054,"judge":89.46264744429882,"hard":95.4967147200157},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":48.29304190430006,"tiers":{"easy":85.3515625,"standard":52.45641838351824,"judge":54.94699646643111,"hard":31.956257594167692},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":95.74545454545455,"tiers":{"easy":100.0,"standard":92.0,"judge":96.0,"hard":96.36363636363636},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":90.39026914339927,"tiers":{"easy":93.8768115942029,"standard":92.1662468513854,"judge":81.55855096882898,"hard":95.25443330763301},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.2916145181476846},"noul":{"n":495,"invalid_rate":0.0505050505050505},"score":{"n":330,"invalid_rate":0.036363636363636376}},"calibration":{"score":95.66103904664365,"parts":{"choice":{"ece_hard":0.04855384588187193,"ece_score":90.28923082362562,"mean_tvd":0.007095392065954789,"n_tvd":39.0,"n":566.0,"n_ece_hard":164.0,"score":94.78984580851507},"noul":{"ece":0.02890577741407529,"brier":0.030226985540601766,"n":470.0,"score":94.21884451718495,"abstention_rate":0.008510638297872353},"score":{"nrps":0.008094957913858556,"top_ece":0.01620825290076149,"n":318.0,"interval90_coverage":0.9779874213836478,"interval90_width":0.01493710691823899,"score":97.97442681423092}}},"speed":{"p50_s_raw":0.6655631910543889,"p95_s_raw":1.9045551142189638,"p50_s_adjusted":1.4811263821087777,"p95_s_adjusted":3.9591102284379276,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.248657697044335,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":47,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":47,"B":46,"C":44}},{"key":"litjev","display":"LitJev (Qwen3.8-27B)","addendum":null,"author":"Zhengxu Yu","repo":"https://github.com/zhengxuyu/litjev","class":"jev-rebuild","licence":"Apache-2.0 (code); Apache-2.0 base weights","open":"yes","underlying":"Qwen/Qwen3.8-27B, frozen, read at the output head","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/litjev/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":58.324628215248765,"calibration":84.45282357688976,"speed":68.2244816164631,"cost":28.35809912639418},"scores":{"A":16.306394195686046,"B":16.74462103865969,"C":15.408465642675305},"views":{"Balanced 33:33:33":14.3890375127424,"Emphasis on Accuracy 60:20:20":15.868297092774801,"Emphasis on Cost 20:20:60":11.689300893996586,"Emphasis on Speed 20:60:20":16.687524645731486,"Intelligence only":18.761441475825645},"composite_ci95":{"A":[16.031824256739885,16.48126738701614],"B":[16.332888760119257,17.052026385960268],"C":[13.385786071253271,16.48126738701614]},"jevbench_score":16.306394195686046,"intelligence":{"I_open":58.061823774277066,"I_sealed":58.58743265622046,"base":58.324628215248765,"gap":-0.5256088819433913,"excess":-5.712236382022634,"penalty":1.0,"per_type_split":{"open|choice":{"cc":80.4361524060561,"tiers":{"easy":98.19113656918904,"standard":90.64205874707565,"judge":82.36514522821577,"hard":69.44770857814335},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":23.485873834650512,"tiers":{"easy":76.47058823529412,"standard":54.54545454545454,"judge":29.41176470588236,"hard":-9.734513274336287},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":70.2634450821246,"tiers":{"easy":54.66237167016875,"standard":81.32857378919948,"judge":70.73340284898273,"hard":68.27868075643254},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":79.03218004271174,"tiers":{"easy":88.28125,"standard":80.98256735340728,"judge":77.47349823321555,"hard":76.91373025516404},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":33.709090909090904,"tiers":{"easy":76.0,"standard":15.999999999999993,"judge":39.99999999999999,"hard":27.27272727272727},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":63.02102701685875,"tiers":{"easy":74.79973205795622,"standard":66.01457217051639,"judge":60.65620381864931,"hard":60.353195578412645},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":84.45282357688976,"parts":{"choice":{"ece_hard":0.04970492013031636,"ece_score":90.05901597393672,"mean_tvd":0.2614886387339129,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":81.95507605027271},"noul":{"ece":0.08802838392984348,"brier":0.09245053724684392,"n":495.0,"score":82.3943232140313,"abstention_rate":0.34545454545454546},"score":{"nrps":0.07045292120710725,"top_ece":0.07468282473279364,"n":330.0,"interval90_coverage":0.9727272727272728,"interval90_width":0.469949494949495,"score":89.00907146636527}}},"speed":{"p50_s_raw":1.4589347330620512,"p95_s_raw":2.3779289319296364,"p50_s_adjusted":3.0678694661241024,"p95_s_adjusted":4.905857863859273,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.2443787068965517,"basis":"price floor: base-model reference price applied"},"rank":48,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":48,"B":48,"C":46}},{"key":"raw-phi-4-mini","display":"Raw Phi-4 mini direct logits","addendum":null,"author":"Microsoft / neutral reproduction","repo":"https://huggingface.co/microsoft/Phi-4-mini-instruct","class":"raw-logit-control","licence":"MIT","open":"yes","underlying":"Phi-4-mini-instruct BF16","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/raw-phi-4-mini/spec.json:raw_adapter.RawControlAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":27.62403248828664,"calibration":71.41052135550255,"speed":89.17438865550707,"cost":53.22051180195924},"scores":{"A":15.222235703588067,"B":13.110562970892774,"C":10.5709970163806},"views":{"Balanced 33:33:33":13.831540696125431,"Emphasis on Accuracy 60:20:20":11.010976074923198,"Emphasis on Cost 20:20:60":14.705353331656267,"Emphasis on Speed 20:60:20":17.219213805677136,"Intelligence only":8.431817920288301},"composite_ci95":{"A":[10.258712918386285,20.737336979219407],"B":[8.602535584899657,18.245435235297943],"C":[7.124106193323811,14.400928457791256]},"jevbench_score":15.222235703588067,"intelligence":{"I_open":38.273825809008926,"I_sealed":19.96000189388855,"base":29.116913851448736,"gap":18.313823915120377,"excess":13.127196415041134,"penalty":0.9487280358495886,"per_type_split":{"open|choice":{"cc":54.09535596329945,"tiers":{"easy":90.95568284594513,"standard":54.76995061086561,"judge":60.580912863070544,"hard":39.678809244026624},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":6.0318962661492606,"tiers":{"easy":64.70588235294117,"standard":-9.090909090909093,"judge":41.176470588235304,"hard":-27.43362831858407},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":54.694225197578064,"tiers":{"easy":24.723976188382103,"standard":70.2841282845953,"judge":59.057398525133834,"hard":51.1194559107016},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":56.668623376411986,"tiers":{"easy":76.56249999999999,"standard":56.53158252207382,"judge":52.296819787985875,"hard":55.04252733900366},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-43.63636363636364,"tiers":{"easy":-36.0,"standard":-36.0,"judge":-43.99999999999999,"hard":-49.09090909090909},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":46.847745941617305,"tiers":{"easy":60.677416027791274,"standard":54.691931112074776,"judge":49.34671366256982,"hard":37.59401004413068},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":71.41052135550255,"parts":{"choice":{"ece_hard":0.15855047915699413,"ece_score":68.28990416860117,"mean_tvd":0.31305162444341283,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":68.49237086212995},"noul":{"ece":0.176680478068443,"brier":0.24340647285719444,"n":495.0,"score":64.6639043863114,"abstention_rate":0.3858585858585859},"score":{"nrps":0.12796697355859052,"top_ece":0.1252636250400414,"n":330.0,"interval90_coverage":0.906060606060606,"interval90_width":0.5098484848484847,"score":81.07528881806633}}},"speed":{"p50_s_raw":0.06535653909668326,"p95_s_raw":0.14041130524128673,"p50_s_adjusted":0.28071307819336655,"p95_s_adjusted":0.4308226104825734,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.03625068965517241,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":49,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":49,"B":50,"C":50}},{"key":"opensourcejev-qwen35-4b-q4km","display":"OpenSourceJev (Qwen3.5-4B Q4_K_M, native llama.cpp)","addendum":null,"author":"sabeel111","repo":"https://github.com/sabeel111/OpenSourceJev","class":"jev-rebuild","licence":"MIT (repository code); Apache-2.0 (Qwen/Qwen3.5-4B base and unsloth/Qwen3.5-4B-GGUF Q4_K_M conversion)","open":"unknown","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/opensourcejev-qwen35-4b-q4km/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":25.769029299779596,"calibration":76.0354906185011,"speed":72.50784557479969,"cost":69.13657649738097},"scores":{"A":13.245465302463623,"B":11.158505729571912,"C":9.198239793377518},"views":{"Balanced 33:33:33":11.882308110018135,"Emphasis on Accuracy 60:20:20":9.179815610247537,"Emphasis on Cost 20:20:60":13.835630815721885,"Emphasis on Speed 20:60:20":14.032252211105229,"Intelligence only":6.84469608018877},"composite_ci95":{"A":[8.979165053864623,18.308517022614893],"B":[7.3854187708120556,15.78442530630126],"C":[6.235531287405989,12.714247932371459]},"jevbench_score":13.245465302463623,"intelligence":{"I_open":27.857284804877462,"I_sealed":23.680773794681734,"base":25.769029299779596,"gap":4.176511010195728,"excess":-1.0101164898835142,"penalty":1.0,"per_type_split":{"open|choice":{"cc":64.38244688758064,"tiers":{"easy":90.95568284594513,"standard":78.1648037431765,"judge":76.14107883817428,"hard":42.02898550724636},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-36.117867903396274,"tiers":{"easy":47.058823529411775,"standard":-30.303030303030297,"judge":-50.98039215686274,"hard":-48.67256637168141},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":55.307275430448016,"tiers":{"easy":27.21197926267279,"standard":76.67532199646642,"judge":50.515131061598964,"hard":55.24118446601941},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":51.48566463280753,"tiers":{"easy":73.63281249999999,"standard":59.2483586144442,"judge":39.045936395759725,"hard":51.39732685297693},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-37.23636363636364,"tiers":{"easy":-28.000000000000004,"standard":-31.999999999999996,"judge":-28.000000000000004,"hard":-49.09090909090909},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":56.793020387601324,"tiers":{"easy":72.95325452898551,"standard":64.46683753148615,"judge":49.08188711036227,"hard":54.69940323824214},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":76.0354906185011,"parts":{"choice":{"ece_hard":0.14908980866425994,"ece_score":70.18203826714802,"mean_tvd":0.3192922165500016,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":69.12640830607393},"noul":{"ece":0.1400313272727273,"brier":0.15711699719054362,"n":495.0,"score":71.99373454545454,"abstention_rate":0.6626262626262627},"score":{"nrps":0.11148831082959457,"top_ece":0.07439255454545449,"n":330.0,"interval90_coverage":0.9242424242424242,"interval90_width":0.4464646464646464,"score":86.98632900397483}}},"speed":{"p50_s_raw":0.6195944650098681,"p95_s_raw":1.9453384675551155,"p50_s_adjusted":1.3891889300197362,"p95_s_adjusted":4.040676935110231,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.0106851539408867,"basis":"price floor: base-model reference price applied; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)"},"rank":50,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":50,"B":51,"C":51}},{"key":"reflex-27b","display":"reflex-27b (Qwen3.8-27B)","addendum":null,"author":"kshetrajna12","repo":"https://github.com/kshetrajna12/reflex","class":"jev-rebuild","licence":"MIT code; Apache-2.0 Qwen weights","open":"yes","underlying":"Qwen3.8-27B, frozen, direct-logit readout averaged across two option orders","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/reflex-27b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":62.794986197868965,"calibration":85.93457237686198,"speed":69.19534736783416,"cost":25.802305452315423},"scores":{"A":13.188297397843682,"B":13.770358241149133,"C":13.188297397843682},"views":{"Balanced 33:33:33":11.556158023973218,"Emphasis on Accuracy 60:20:20":13.185620031521559,"Emphasis on Cost 20:20:60":9.079850594306789,"Emphasis on Speed 20:60:20":13.581847895497212,"Intelligence only":16.722530048872894},"composite_ci95":{"A":[12.992358682861784,13.300762882643161],"B":[13.473043870540051,13.98155386873936],"C":[12.664340288901577,13.300762882643161]},"jevbench_score":13.188297397843682,"intelligence":{"I_open":63.0906276726272,"I_sealed":62.499344723110724,"base":62.794986197868965,"gap":0.5912829495164758,"excess":-4.595344550562767,"penalty":1.0,"per_type_split":{"open|choice":{"cc":84.80373580586229,"tiers":{"easy":98.19113656918904,"standard":85.96308812061348,"judge":93.7759336099585,"hard":74.14806110458284},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":30.922026091208803,"tiers":{"easy":70.58823529411764,"standard":15.15151515151516,"judge":58.823529411764696,"hard":7.964601769911495},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":73.54612112081051,"tiers":{"easy":48.2951624423963,"standard":84.68773586572438,"judge":76.40678899082567,"hard":72.14255251544573},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":80.29589093906678,"tiers":{"easy":88.28125,"standard":86.41611953814805,"judge":74.82332155477033,"hard":79.34386391251519},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":35.709090909090904,"tiers":{"easy":68.0,"standard":24.0,"judge":43.99999999999999,"hard":27.27272727272727},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":71.49305232117449,"tiers":{"easy":82.4410579710145,"standard":74.76243366918558,"judge":67.25852569502949,"hard":70.29725520431766},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":85.93457237686198,"parts":{"choice":{"ece_hard":0.03788619494584862,"ece_score":92.42276101083029,"mean_tvd":0.2503876265500017,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":83.69199917791505},"noul":{"ece":0.09199937373737374,"brier":0.08261424794053736,"n":495.0,"score":81.60012525252525,"abstention_rate":0.31919191919191914},"score":{"nrps":0.06417932175466212,"top_ece":0.04279441212121212,"n":330.0,"interval90_coverage":0.9757575757575757,"interval90_width":0.3825757575757574,"score":92.51159270014568}}},"speed":{"p50_s_raw":1.2991375599522144,"p95_s_raw":2.1146518142428246,"p50_s_adjusted":2.7482751199044286,"p95_s_adjusted":4.3793036284856495,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.2973421551724138,"basis":"price floor: base-model reference price applied"},"rank":51,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":51,"B":49,"C":49}},{"key":"open-jev-zefan-2b","display":"Open-Jev 2B (Zefan Cai)","addendum":null,"author":"Zefan Cai (@Zefan_Cai)","repo":"https://github.com/Zefan-Cai/Open-Jev","class":"jev-rebuild","licence":"MIT (loader); Apache-2.0 (adapter and pinned Qwen base); CC0-1.0 public training projection","open":"yes","underlying":"Qwen3.5-2B plus rank-8 LoRA and trained scalar decision head","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/open-jev-zefan-2b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":33.564173076571095,"calibration":73.62274281306851,"speed":75.76421649699464,"cost":33.05490478272432},"scores":{"A":9.072856494116836,"B":8.443734103226019,"C":6.300594787581135},"views":{"Balanced 33:33:33":8.066510781336797,"Emphasis on Accuracy 60:20:20":7.413258997675015,"Emphasis on Cost 20:20:60":7.362375355630084,"Emphasis on Speed 20:60:20":9.8825122020589,"Intelligence only":6.610277008881134},"composite_ci95":{"A":[6.618399852202896,11.536977019372797],"B":[6.032267746205537,10.927897940214102],"C":[4.5961110084742325,8.011789596786665]},"jevbench_score":9.072856494116836,"intelligence":{"I_open":38.78572545166409,"I_sealed":28.34262070147809,"base":33.564173076571095,"gap":10.443104750186002,"excess":5.25647725010676,"penalty":1.0,"per_type_split":{"open|choice":{"cc":51.51924767992967,"tiers":{"easy":87.33795598432319,"standard":51.650636859890824,"judge":71.99170124481329,"hard":27.144535840188002},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":17.336141214329654,"tiers":{"easy":76.47058823529412,"standard":9.090909090909083,"judge":39.2156862745098,"hard":-9.734513274336287},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":47.50178746073294,"tiers":{"easy":35.09794173421018,"standard":65.72344820125355,"judge":48.51920193657173,"hard":40.72885766522424},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":45.4862239976945,"tiers":{"easy":70.703125,"standard":57.889970568258995,"judge":40.37102473498235,"hard":36.816524908869994},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-8.14545454545455,"tiers":{"easy":19.999999999999996,"standard":-12.0,"judge":-4.0000000000000036,"hard":-16.36363636363637},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":47.68709265219432,"tiers":{"easy":60.087358156308326,"standard":58.822783994022785,"judge":44.278631641034735,"hard":41.575526363621286},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":73.62274281306851,"parts":{"choice":{"ece_hard":0.13443745135490978,"ece_score":73.11250972901804,"mean_tvd":0.31141793250204397,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":70.98535823940682},"noul":{"ece":0.15698662520519996,"brier":0.21726970854511088,"n":495.0,"score":68.60267495896001,"abstention_rate":0.29292929292929293},"score":{"nrps":0.13193684670715047,"top_ece":0.12122962423803765,"n":330.0,"interval90_coverage":0.9303030303030303,"interval90_width":0.5454545454545454,"score":81.28019524083871}}},"speed":{"p50_s_raw":0.42904718848876655,"p95_s_raw":1.2403676671441641,"p50_s_adjusted":1.008094376977533,"p95_s_adjusted":2.630735334288328,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.17041317733990147,"basis":"documented hosted-model estimate"},"rank":52,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":52,"B":52,"C":53}},{"key":"gliner2-large","display":"GLiNER2 large (Fastino)","addendum":null,"author":"Fastino","repo":"https://huggingface.co/fastino/gliner2-large-v1","class":"classifier","licence":"Apache-2.0","open":"yes","underlying":"GLiNER2 large schema-conditioned extractor (DeBERTa-v3-large encoder)","endpoint_kind":"cpu","endpoint_condition":"evaluator-owned CPU container, offline read-only, pinned cores","gpu":"cpu","adapter_id":"images/gliner2-large/spec.json:warm_adapter.WarmLoaded","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":22.49372087756688,"calibration":42.37995459053843,"speed":64.775339304984,"cost":77.5922354035203},"scores":{"A":8.399649998514048,"B":7.1852200696572694,"C":5.833090276745868},"views":{"Balanced 33:33:33":8.342104237606092,"Emphasis on Accuracy 60:20:20":6.258239920284327,"Emphasis on Cost 20:20:60":10.267354778838929,"Emphasis on Speed 20:60:20":9.762184401986273,"Intelligence only":4.552436497562072},"composite_ci95":{"A":[5.206602086900395,12.336461805804452],"B":[4.351719703950813,10.741887051792189],"C":[3.615695893680829,8.56698736514198]},"jevbench_score":8.399649998514048,"intelligence":{"I_open":26.030302249344764,"I_sealed":18.957139505788998,"base":22.49372087756688,"gap":7.073162743555766,"excess":1.8865352434765237,"penalty":1.0,"per_type_split":{"open|choice":{"cc":26.574056711930325,"tiers":{"easy":87.33795598432319,"standard":34.4944112295295,"judge":20.124481327800847,"hard":12.260086173129631},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":16.353067373369296,"tiers":{"easy":64.70588235294117,"standard":3.0303030303030276,"judge":50.98039215686274,"hard":-15.04424778761062},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":35.163782662734675,"tiers":{"easy":8.23002435297282,"standard":62.58297647158422,"judge":35.395996727022364,"hard":28.013464787534602},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":28.41569323833785,"tiers":{"easy":35.54687499999999,"standard":32.08059769074034,"judge":20.49469964664312,"hard":30.741190765492117},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-0.6181818181818213,"tiers":{"easy":-36.0,"standard":4.0000000000000036,"judge":0.0,"hard":5.454545454545445},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":29.073907097210963,"tiers":{"easy":33.32924379728206,"standard":30.537722630646826,"judge":22.797688869828836,"hard":31.98532882601185},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":42.37995459053843,"parts":{"choice":{"ece_hard":0.35899151845529187,"ece_score":28.201696308941628,"mean_tvd":0.44204858856583795,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":41.99841872617891},"noul":{"ece":0.34577794089535685,"brier":0.35859397977345603,"n":495.0,"score":30.84441182092863,"abstention_rate":0.10303030303030303},"score":{"nrps":0.20017276385688332,"top_ece":0.3569432858264807,"n":330.0,"interval90_coverage":0.6909090909090909,"interval90_width":0.3994949494949492,"score":54.29703322450776}}},"speed":{"p50_s_raw":0.8647058844799176,"p95_s_raw":8.784600352274692,"p50_s_adjusted":1.879411768959835,"p95_s_adjusted":17.719200704549383,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.005583743842364532,"basis":"ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies"},"rank":53,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":53,"B":54,"C":54}},{"key":"qwen3-reranker-4b","display":"Qwen3-Reranker-4B","addendum":null,"author":"Qwen","repo":"https://huggingface.co/Qwen/Qwen3-Reranker-4B","class":"reranker","licence":"Apache-2.0","open":"yes","underlying":"4B instruction-aware generative reranker","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/qwen3-reranker-4b/spec.json:reranker_adapter.RerankerAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":21.025680453051585,"calibration":76.2573027508077,"speed":79.60936519080639,"cost":48.39770052432463},"scores":{"A":7.0578732911112105,"B":5.856117186678418,"C":4.901300896605006},"views":{"Balanced 33:33:33":6.152668509566967,"Emphasis on Accuracy 60:20:20":4.709324078073576,"Emphasis on Cost 20:20:60":6.78411455677089,"Emphasis on Speed 20:60:20":7.821945004625668,"Intelligence only":3.4835305028101717},"composite_ci95":{"A":[4.256683239685833,10.735647697538731],"B":[3.4274523885192507,9.149689058545547],"C":[2.956030027559606,7.455310901068563]},"jevbench_score":7.0578732911112105,"intelligence":{"I_open":30.348285824317287,"I_sealed":13.363369857977867,"base":21.855827841147576,"gap":16.984915966339422,"excess":11.798288466260178,"penalty":0.9620171153373982,"per_type_split":{"open|choice":{"cc":56.69813267337053,"tiers":{"easy":89.14681941513415,"standard":64.12789186378996,"judge":74.06639004149378,"hard":31.84488836662748},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-10.056946350543434,"tiers":{"easy":17.647058823529417,"standard":-42.42424242424242,"judge":37.254901960784316,"hard":-36.283185840707965},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":44.40367115012478,"tiers":{"easy":13.096620724978504,"standard":54.28179029670248,"judge":42.537424561879064,"hard":48.69105912430677},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":45.59936268245455,"tiers":{"easy":67.7734375,"standard":41.58931401403669,"judge":50.97173144876326,"hard":38.031591737545575},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-45.92727272727273,"tiers":{"easy":-43.99999999999999,"standard":-40.0,"judge":-56.00000000000001,"hard":-41.81818181818182},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":40.418019618751785,"tiers":{"easy":55.03666193584841,"standard":42.80674647447732,"judge":36.240240687009674,"hard":38.702329810421446},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":76.2573027508077,"parts":{"choice":{"ece_hard":0.09906655742890301,"ece_score":80.1866885142194,"mean_tvd":0.25225164327791294,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":77.48076209321405},"noul":{"ece":0.16823664778156078,"brier":0.21948898953258267,"n":495.0,"score":66.35267044368784,"abstention_rate":0.49696969696969695},"score":{"nrps":0.14741060021939884,"top_ece":0.07690994273508868,"n":330.0,"interval90_coverage":0.9272727272727272,"interval90_width":0.6116161616161616,"score":84.9384757155212}}},"speed":{"p50_s_raw":0.18062184238806367,"p95_s_raw":0.9950535912532349,"p50_s_adjusted":0.5112436847761274,"p95_s_adjusted":2.14010718250647,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.05249000923645321,"basis":"ESTIMATE: hosted exact-model reference (deepinfra:Qwen/Qwen3-Reranker-4B); no exact base-model floor applies"},"rank":54,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":54,"B":55,"C":55}},{"key":"deepseek-flash","display":"DeepSeek V4.1 Flash (thinking default)","addendum":null,"author":"DeepSeek","repo":null,"class":"llm-baseline","licence":"open weights, proprietary API route","open":"weights","underlying":"DeepSeek-V4.1-Flash","endpoint_kind":"api","endpoint_condition":"the operator's hosted API","gpu":null,"adapter_id":"harness/specs/deepseek-flash.json:jevbench.adapters.openai_compat.OpenAICompatAdapter","api_flag":true,"api_exposure_note":"API measurement: the operator's endpoint received sealed item text, without answers.","status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1587},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":93.69043763543172,"calibration":96.91785139608032,"speed":69.39902343452796,"cost":19.093609002972528},"scores":{"A":6.645505039231519,"B":7.406272143196756,"C":6.645505039231519},"views":{"Balanced 33:33:33":5.648067608779863,"Emphasis on Accuracy 60:20:20":7.3796291044149225,"Emphasis on Cost 20:20:60":4.001744057440232,"Emphasis on Speed 20:60:20":6.860785464950723,"Intelligence only":13.662535665508406},"composite_ci95":{"A":[6.624148909040629,6.663488366039578],"B":[7.368735537079963,7.438725543018248],"C":[6.624148909040629,6.663488366039578]},"jevbench_score":6.645505039231519,"intelligence":{"I_open":91.84671609723216,"I_sealed":95.53415917363128,"base":93.69043763543172,"gap":-3.68744307639912,"excess":-8.874070576478363,"penalty":1.0,"per_type_split":{"open|choice":{"cc":96.20131099896392,"tiers":{"easy":96.38227313837805,"standard":93.76137249805042,"judge":97.9253112033195,"hard":96.08303956130042},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":86.17465650781632,"tiers":{"easy":100.0,"standard":93.93939393939394,"judge":86.27450980392157,"hard":78.76106194690264},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":93.16418078491628,"tiers":{"easy":100.0,"standard":90.99779151943463,"judge":93.83879423328965,"hard":92.03246052760616},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":98.23289353352598,"tiers":{"easy":100.0,"standard":100.0,"judge":97.34982332155477,"hard":97.56986634264887},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":96.54545454545455,"tiers":{"easy":92.0,"standard":100.0,"judge":96.0,"hard":96.36363636363636},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":91.82412944191336,"tiers":{"easy":99.83695652173913,"standard":97.22292191435768,"judge":94.2005054759899,"hard":85.33924441017734},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.020025031289111372},"noul":{"n":495,"invalid_rate":0.018181818181818188},"score":{"n":330,"invalid_rate":0.036363636363636376}},"calibration":{"score":96.91785139608032,"parts":{"choice":{"ece_hard":0.04581060173808424,"ece_score":90.83787965238315,"mean_tvd":0.012719194915753415,"n_tvd":47.0,"n":783.0,"n_ece_hard":270.0,"score":94.7829800804039},"noul":{"ece":0.01675268517489696,"brier":0.019247259932996506,"n":486.0,"score":96.6494629650206,"abstention_rate":0.012345679012345734},"score":{"nrps":0.00022347944975023944,"top_ece":0.006677148846960353,"n":318.0,"interval90_coverage":1.0,"interval90_width":0.0047169811320754715,"score":99.32111114281645}}},"speed":{"p50_s_raw":1.7762285992503166,"p95_s_raw":6.465450605377541,"p50_s_adjusted":1.7762285992503166,"p95_s_adjusted":6.465450605377541,"n":450,"adjustment":"none (hosted API, measured as is)"},"cost":{"kind":"estimate","usd_per_1000":0.4975992302955664,"basis":"ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)"},"rank":55,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":55,"B":53,"C":52}},{"key":"simplejev-qwen3.5-0.8b","display":"SimpleJev (Qwen3.5-0.8B, CPU)","addendum":null,"author":"sabeel111 / Featherless AI","repo":"https://github.com/featherless-ai/simple-jev","class":"jev-rebuild","licence":"Apache-2.0 server; Apache-2.0 Qwen3.5-0.8B checkpoint","open":"yes","underlying":"Qwen3.5-0.8B through SimpleJev native assistant-prefill option-logit scorer","endpoint_kind":"cpu","endpoint_condition":"evaluator-owned CPU container, offline read-only, pinned cores","gpu":"cpu","adapter_id":"images/simplejev-qwen3.5-0.8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1617},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":16.7457779502415,"calibration":46.67276047458846,"speed":59.07352652626999,"cost":70.31175519898045},"scores":{"A":3.995571708038271,"B":3.2605340633066584,"C":2.7747025750265775},"views":{"Balanced 33:33:33":3.7032739070208662,"Emphasis on Accuracy 60:20:20":2.66686632240896,"Emphasis on Cost 20:20:60":4.700645988538198,"Emphasis on Speed 20:60:20":4.496698803590565,"Intelligence only":1.8783476496641063},"composite_ci95":{"A":[2.0618884680283314,6.661268630105466],"B":[1.6182691638588302,5.613466175208515],"C":[1.431866991686341,4.625880993128795]},"jevbench_score":3.995571708038271,"intelligence":{"I_open":13.326965757246013,"I_sealed":20.164590143236985,"base":16.7457779502415,"gap":-6.837624385990972,"excess":-12.024251886070214,"penalty":1.0,"per_type_split":{"open|choice":{"cc":28.275464895018914,"tiers":{"easy":85.52909255351221,"standard":34.4944112295295,"judge":19.0871369294606,"hard":17.74383078730903},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-12.834066852807085,"tiers":{"easy":-5.882352941176472,"standard":-3.0303030303030276,"judge":-47.05882352941176,"hard":6.194690265486735},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":24.539499229526214,"tiers":{"easy":-1.6354458020304774,"standard":23.475110530688738,"judge":16.31938444589648,"hard":37.780515924556425},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":27.771564822999164,"tiers":{"easy":44.335937499999986,"standard":28.00543355218476,"judge":31.09540636042403,"hard":21.0206561360875},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":4.254545454545452,"tiers":{"easy":36.00000000000001,"standard":12.00000000000001,"judge":15.999999999999993,"hard":-16.36363636363637},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":28.46766015216634,"tiers":{"easy":40.82324942776366,"standard":35.51984055110081,"judge":24.57770993391708,"hard":24.770135297486707},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.007509386733416723},"noul":{"n":495,"invalid_rate":0.002020202020202033},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":46.67276047458846,"parts":{"choice":{"ece_hard":0.20237649586807754,"ece_score":59.52470082638449,"mean_tvd":0.29457199268122575,"n_tvd":50.0,"n":793.0,"n_ece_hard":271.0,"score":65.03375077913095},"noul":{"ece":0.5196194263761825,"brier":0.5189770017458518,"n":494.0,"score":0.0,"abstention_rate":0.0},"score":{"nrps":0.2023654034513244,"top_ece":0.1489719918279937,"n":330.0,"interval90_coverage":0.8575757575757575,"interval90_width":0.6909090909090907,"score":74.98453064463442}}},"speed":{"p50_s_raw":3.6104603729909286,"p95_s_raw":8.321448143641462,"p50_s_adjusted":7.3709207459818575,"p95_s_adjusted":16.79289628728292,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.009763559113300492,"basis":"documented hosted-model estimate"},"rank":56,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":56,"B":57,"C":57}},{"key":"simplejev-qwen3.8-27b","display":"SimpleJev Qwen3.8-27B","addendum":null,"author":"Featherless AI","repo":"https://github.com/featherless-ai/simple-jev","class":"jev-rebuild","licence":"Apache-2.0 (Qwen weights); repository licence not stated","open":"yes","underlying":"Qwen3.8-27B through SimpleJev's direct-logit classifier","endpoint_kind":"demo","endpoint_condition":"the author's public demo endpoint","gpu":null,"adapter_id":"harness/specs/simplejev-qwen3.8-27b.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":true,"api_exposure_note":"API measurement: the operator's endpoint received sealed item text, without answers.","status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":72.75482343968588,"calibration":87.1969052856583,"speed":74.91530661775406,"cost":14.895025990507506},"scores":{"A":3.3584255994370715,"B":3.7149465104642747,"C":3.3584255994370715},"views":{"Balanced 33:33:33":2.8253739760349332,"Emphasis on Accuracy 60:20:20":3.645466466370485,"Emphasis on Cost 20:20:60":1.9418744899793077,"Emphasis on Speed 20:60:20":3.669365485962498,"Intelligence only":6.456606413207646},"composite_ci95":{"A":[3.3298853690492023,3.3731363629589097],"B":[3.6686384808568335,3.7449217201789837],"C":[3.3298853690492023,3.3731363629589097]},"jevbench_score":3.3584255994370715,"intelligence":{"I_open":71.90840452793111,"I_sealed":73.60124235144063,"base":72.75482343968588,"gap":-1.692837823509521,"excess":-6.8794653235887635,"penalty":1.0,"per_type_split":{"open|choice":{"cc":85.42687050046183,"tiers":{"easy":98.19113656918904,"standard":87.52274499610087,"judge":94.81327800829875,"hard":74.14806110458284},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":51.10091019513195,"tiers":{"easy":82.35294117647058,"standard":60.606060606060595,"judge":58.823529411764696,"hard":32.74336283185841},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":79.19743288819956,"tiers":{"easy":79.17508201475533,"standard":84.86964706071151,"judge":82.73204775623967,"hard":73.71595236927453},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":80.67343761665602,"tiers":{"easy":88.28125,"standard":82.34095539959247,"judge":78.79858657243817,"hard":79.34386391251519},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":60.94545454545455,"tiers":{"easy":76.0,"standard":63.99999999999999,"judge":60.00000000000001,"hard":56.363636363636374},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":79.18483489221134,"tiers":{"easy":82.34156829293816,"standard":78.53194588489751,"judge":79.57425039229116,"hard":78.4300344206267},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":87.1969052856583,"parts":{"choice":{"ece_hard":0.03969125406621597,"ece_score":92.0617491867568,"mean_tvd":0.3276173629231819,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":79.65000644721931},"noul":{"ece":0.0371275107796666,"brier":0.0685308410661177,"n":495.0,"score":92.57449784406668,"abstention_rate":0.19393939393939397},"score":{"nrps":0.043365328698605106,"top_ece":0.08465521999380805,"n":330.0,"interval90_coverage":0.990909090909091,"interval90_width":0.2717171717171717,"score":89.36621156568894}}},"speed":{"p50_s_raw":0.8395581245422363,"p95_s_raw":0.9601930942386389,"p50_s_adjusted":1.6791162490844727,"p95_s_adjusted":1.9203861884772777,"n":450,"adjustment":"x2 demo-endpoint adjustment (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.686803448275862,"basis":"ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)"},"rank":57,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":57,"B":56,"C":56}},{"key":"decision-machine-1","display":"decision-machine-1 (milliseconds.ai)","addendum":null,"author":"milliseconds.ai (Baptiste Laget)","repo":"https://www.milliseconds.ai","class":"decision-api","licence":"proprietary API, closed weights","open":"no","underlying":"decision-machine-1 (weights not published)","endpoint_kind":"api","endpoint_condition":"the operator's hosted API","gpu":null,"adapter_id":"harness/specs/decision-machine-1.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":true,"api_exposure_note":"API measurement: the operator's endpoint received sealed item text, without answers.","status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":14.816438176464153,"calibration":81.18169307356939,"speed":92.77166935344088,"cost":56.297500611451596},"scores":{"A":3.2416665489183343,"B":2.4968194273457973,"C":2.2511573256377324},"views":{"Balanced 33:33:33":2.7430986994891735,"Emphasis on Accuracy 60:20:20":1.9005028454194188,"Emphasis on Cost 20:20:60":3.3372862355103208,"Emphasis on Speed 20:60:20":3.7336769644480805,"Intelligence only":1.3010423425783209},"composite_ci95":{"A":[1.730049026826611,5.392829369228498],"B":[1.2909980314827887,4.283648012064621],"C":[1.2014229352962575,3.745020395297569]},"jevbench_score":3.2416665489183343,"intelligence":{"I_open":27.50038667381044,"I_sealed":5.129340447229145,"base":16.314863560519793,"gap":22.371046226581292,"excess":17.18441872650205,"penalty":0.9081558127349795,"per_type_split":{"open|choice":{"cc":53.2654927384921,"tiers":{"easy":89.14681941513415,"standard":53.21029373537822,"judge":71.99170124481329,"hard":30.27810419114765},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-17.549256227028224,"tiers":{"easy":52.941176470588225,"standard":-51.515151515151516,"judge":37.254901960784316,"hard":-59.29203539823009},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":46.78492350996744,"tiers":{"easy":29.733926322635995,"standard":61.068096883467916,"judge":44.82424292253913,"hard":45.37659656062129},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":44.942650158916166,"tiers":{"easy":64.84374999999999,"standard":48.381254244962655,"judge":40.37102473498235,"hard":41.67679222357231},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-67.96363636363635,"tiers":{"easy":-43.99999999999999,"standard":-68.0,"judge":-72.0,"hard":-70.9090909090909},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":38.40900754640762,"tiers":{"easy":51.11240578541303,"standard":49.00817958829449,"judge":34.66912890206911,"hard":32.73848094896671},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":81.18169307356939,"parts":{"choice":{"ece_hard":0.08149957836600445,"ece_score":83.7000843267991,"mean_tvd":0.23287647233513212,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":80.20621854664294},"noul":{"ece":0.10654949494949495,"brier":0.2044606949494951,"n":495.0,"score":78.69010101010102,"abstention_rate":0.6060606060606061},"score":{"nrps":0.14849255922750398,"top_ece":0.07926612374660588,"n":330.0,"interval90_coverage":0.906060606060606,"interval90_width":0.6166666666666666,"score":84.64875966396421}}},"speed":{"p50_s_raw":0.18023579940199852,"p95_s_raw":0.29308393076062134,"p50_s_adjusted":0.18023579940199852,"p95_s_adjusted":0.29308393076062134,"n":450,"adjustment":"none (hosted API, measured as is)"},"cost":{"kind":"estimate","usd_per_1000":0.02862529556650246,"basis":"operator standard launch list price (interpretation I-1); no exact base-model floor applies"},"rank":58,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":58,"B":58,"C":58}},{"key":"gliner2.5-multi","display":"GLiNER2.5 multi (Fastino, 287M)","addendum":null,"author":"Fastino","repo":"https://huggingface.co/fastino/gliner2.5-multi-v1","class":"classifier","licence":"Apache-2.0","open":"yes","underlying":"GLiNER2.5 multilingual schema-conditioned extractor, 287M","endpoint_kind":"cpu","endpoint_condition":"evaluator-owned CPU container, offline read-only, pinned cores","gpu":"cpu","adapter_id":"images/gliner2.5-multi/spec.json:warm_adapter.WarmLoaded","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":13.995189304514808,"calibration":57.96384531072885,"speed":66.5730877741724,"cost":86.62313527343974},"scores":{"A":2.7186855157112975,"B":2.0979157487811895,"C":1.88797605257729},"views":{"Balanced 33:33:33":2.3978989928514545,"Emphasis on Accuracy 60:20:20":1.6259462759138177,"Emphasis on Cost 20:20:60":3.234583320343537,"Emphasis on Speed 20:60:20":3.0589465345482507,"Intelligence only":1.096468913176239},"composite_ci95":{"A":[1.196852251293987,5.101047430526342],"B":[0.8901107605029109,4.071135024636648],"C":[0.8311473967319353,3.542394048976626]},"jevbench_score":2.7186855157112975,"intelligence":{"I_open":15.77937087068345,"I_sealed":12.211007738346165,"base":13.995189304514808,"gap":3.5683631323372857,"excess":-1.6182643677419568,"penalty":1.0,"per_type_split":{"open|choice":{"cc":19.436913892915463,"tiers":{"easy":78.29363883026832,"standard":29.815440603067323,"judge":5.601659751037364,"hard":9.909909909909892},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":0.15995456911646766,"tiers":{"easy":23.529411764705888,"standard":-9.090909090909093,"judge":23.529411764705888,"hard":-18.58407079646017},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":27.741244150018424,"tiers":{"easy":-5.203924413066585,"standard":42.01767546895648,"judge":33.347933189662704,"hard":24.634303851587436},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":17.23700918907234,"tiers":{"easy":32.61718749999999,"standard":10.346388951777246,"judge":16.519434628975272,"hard":17.375455650060772},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-2.5818181818181833,"tiers":{"easy":4.0000000000000036,"standard":-4.0000000000000036,"judge":0.0,"hard":-5.454545454545457},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":21.977832207784342,"tiers":{"easy":32.451026939341574,"standard":22.802673394719065,"judge":13.23783973949041,"hard":25.502107282648122},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":57.96384531072885,"parts":{"choice":{"ece_hard":0.19331665541505985,"ece_score":61.33666891698803,"mean_tvd":0.3324832180692498,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":64.04417355503152},"noul":{"ece":0.3081204742930519,"brier":0.3316772483381491,"n":495.0,"score":38.37590514138962,"abstention_rate":0.18585858585858583},"score":{"nrps":0.20624840028309196,"top_ece":0.18216122750079994,"n":330.0,"interval90_coverage":0.896969696969697,"interval90_width":0.8267676767676767,"score":71.47145723576541}}},"speed":{"p50_s_raw":0.6099890284240246,"p95_s_raw":7.9592920757830115,"p50_s_adjusted":1.369978056848049,"p95_s_adjusted":16.06858415156602,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.002791871921182266,"basis":"ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies"},"rank":59,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":59,"B":59,"C":59}},{"key":"gliner2","display":"GLiNER2 (Fastino, gliner2.5-base)","addendum":null,"author":"Fastino","repo":"https://github.com/fastino-ai/GLiNER2","class":"classifier","licence":"Apache-2.0","open":"yes","underlying":"DeBERTa-v3-base schema-conditioned extractor (GLiNER2.5 boundary architecture), 194M","endpoint_kind":"cpu","endpoint_condition":"evaluator-owned CPU container, offline read-only, pinned cores","gpu":"cpu","adapter_id":"images/gliner2/spec.json:warm_adapter.WarmLoaded","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":13.475160169985848,"calibration":35.566185234425355,"speed":70.33059310938175,"cost":86.62313527343974},"scores":{"A":2.2681537932105402,"B":1.7951492831599238,"C":1.575106800840653},"views":{"Balanced 33:33:33":2.1795380936293123,"Emphasis on Accuracy 60:20:20":1.462027588988605,"Emphasis on Cost 20:20:60":2.951032720407862,"Emphasis on Speed 20:60:20":2.8281151639239033,"Intelligence only":0.9787275187231775},"composite_ci95":{"A":[0.9029935487473513,4.50206881847512],"B":[0.682640869013673,3.698158199520721],"C":[0.6270788532967718,3.1264366794966114]},"jevbench_score":2.2681537932105402,"intelligence":{"I_open":17.24017625553533,"I_sealed":9.710144084436365,"base":13.475160169985848,"gap":7.530032171098966,"excess":2.343404671019724,"penalty":1.0,"per_type_split":{"open|choice":{"cc":24.79595717384202,"tiers":{"easy":81.91136569189027,"standard":40.73303873147908,"judge":8.71369294605811,"hard":14.61026243634937},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":1.5045825248844507,"tiers":{"easy":41.176470588235304,"standard":-21.212121212121215,"judge":25.49019607843137,"hard":-15.04424778761062},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":25.419989067879534,"tiers":{"easy":3.1212615250359788,"standard":53.252372656757444,"judge":27.812826480825635,"hard":15.283851099441891},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":20.989344151513734,"tiers":{"easy":32.61718749999999,"standard":28.00543355218476,"judge":20.49469964664312,"hard":14.94532199270962},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-10.0,"tiers":{"easy":28.000000000000004,"standard":-24.0,"judge":0.0,"hard":-19.999999999999996},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":18.14108810179536,"tiers":{"easy":31.15651420104033,"standard":19.75534412338449,"judge":9.97925546483749,"hard":20.20147804390796},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":35.566185234425355,"parts":{"choice":{"ece_hard":0.4157658562010376,"ece_score":16.846828759792476,"mean_tvd":0.4761788956904523,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":34.61446959537363},"noul":{"ece":0.3247925844352796,"brier":0.34914475920123894,"n":495.0,"score":35.041483112944086,"abstention_rate":0.18181818181818177},"score":{"nrps":0.25914794010083303,"top_ece":0.502127007914312,"n":330.0,"interval90_coverage":0.5121212121212121,"interval90_width":0.22702020202020204,"score":37.04260299495835}}},"speed":{"p50_s_raw":0.4240573085844517,"p95_s_raw":4.567268735170364,"p50_s_adjusted":0.9981146171689034,"p95_s_adjusted":9.284537470340728,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.002791871921182266,"basis":"ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies"},"rank":60,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":60,"B":60,"C":60}},{"key":"jevact","display":"JevAct (einptein, jev1-2b-v2)","addendum":null,"author":"einptein","repo":"https://github.com/fstandhartinger/jevbench/issues/66","class":"jev-rebuild","licence":"not stated (the author published only an endpoint and the evaluation client)","open":"no","underlying":"the author's own checkpoint model/jev1-2b-v2/checkpoint-45000-67.53 over a Qwen3.5-2B base, served by the author on his own machine; one request per decision returns a native probability for every offered option","endpoint_kind":"demo","endpoint_condition":"the author's public demo endpoint","gpu":null,"adapter_id":"harness/specs/jevact.json:jevbench.adapters.jevact_api.JevActApiAdapter","api_flag":true,"api_exposure_note":"API measurement: the operator's endpoint received sealed item text, without answers.","status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1496},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":11.23787871976056,"calibration":62.54294670001093,"speed":76.43004589872001,"cost":68.56133553360087},"scores":{"A":1.523364886742046,"B":1.139657303464623,"C":1.0578922824597539},"views":{"Balanced 33:33:33":1.2991218913085527,"Emphasis on Accuracy 60:20:20":0.8572965832446319,"Emphasis on Cost 20:20:60":1.7320742174222945,"Emphasis on Speed 20:60:20":1.768496074540878,"Intelligence only":0.5676923133446331},"composite_ci95":{"A":[0.5244924655946405,3.1790628916081416],"B":[0.37525621735250697,2.472834711495382],"C":[0.36423087888516703,2.207682563616765]},"jevbench_score":1.523364886742046,"intelligence":{"I_open":19.718374313549337,"I_sealed":3.4677959832723193,"base":11.593085148410829,"gap":16.250578330277015,"excess":11.063950830197772,"penalty":0.9693604916980223,"per_type_split":{"open|choice":{"cc":31.064163034954287,"tiers":{"easy":85.52909255351221,"standard":45.41200935794125,"judge":29.460580912863083,"hard":11.476694085389722},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-10.650545012856309,"tiers":{"easy":29.41176470588236,"standard":-21.212121212121215,"judge":31.372549019607842,"hard":-46.90265486725663},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":38.741504918550035,"tiers":{"easy":10.57001976596731,"standard":49.03532322866402,"judge":47.75306339910301,"hard":33.87879819122398},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":35.28176715628971,"tiers":{"easy":56.054687499999986,"standard":32.08059769074034,"judge":28.44522968197881,"hard":36.816524908869994},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-55.345454545454544,"tiers":{"easy":-52.0,"standard":-60.0,"judge":-52.0,"hard":-56.363636363636374},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":30.467075338981786,"tiers":{"easy":51.70725102450335,"standard":34.736082929723736,"judge":28.810676983883265,"hard":24.264826388554305},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.12515644555694616},"noul":{"n":495,"invalid_rate":0.0444444444444444},"score":{"n":330,"invalid_rate":0.018181818181818188}},"calibration":{"score":62.54294670001093,"parts":{"choice":{"ece_hard":0.21664219000489238,"ece_score":56.67156199902153,"mean_tvd":0.35928157130262306,"n_tvd":42.0,"n":699.0,"n_ece_hard":207.0,"score":60.371702434379614},"noul":{"ece":0.23128113451258264,"brier":0.29823669613993214,"n":473.0,"score":53.74377309748347,"abstention_rate":0.39746300211416485},"score":{"nrps":0.1749893478993819,"top_ece":0.17737168036861187,"n":324.0,"interval90_coverage":0.8950617283950617,"interval90_width":0.6769547325102881,"score":73.51336456816972}}},"speed":{"p50_s_raw":0.4011184424161911,"p95_s_raw":1.4179561108350751,"p50_s_adjusted":0.8022368848323822,"p95_s_adjusted":2.8359122216701502,"n":450,"adjustment":"x2 demo-endpoint adjustment (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.011167487684729064,"basis":"ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies"},"rank":61,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":61,"B":62,"C":61}},{"key":"clm-8b","display":"CLM-8B (Contrastive-LM, clm-latest)","addendum":null,"author":"Contrastive-LM (Kwok, Kang, Suresh, Saad-Falcon, Pavone, Ré, Mirhoseini)","repo":"https://github.com/Contrastive-LM/CLM","class":"system-one-open","licence":"Apache-2.0 (code and CLM-v0.1-8B head); Qwen3-8B encoder Apache-2.0","open":"yes","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container","gpu":"RTXPRO6000","adapter_id":"images/clm-8b/spec.json:clm_adapter.CLMAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":11.4306380303076,"calibration":48.48351103482105,"speed":93.28574041500174,"cost":50.501662240594015},"scores":{"A":1.5079991770819052,"B":1.1556905102757526,"C":1.0472216507513232},"views":{"Balanced 33:33:33":1.328681296240889,"Emphasis on Accuracy 60:20:20":0.8919539241243533,"Emphasis on Cost 20:20:60":1.6580320036736744,"Emphasis on Speed 20:60:20":1.8739962877475922,"Intelligence only":0.5974085148705298},"composite_ci95":{"A":[0.4792660492923253,3.1684642573866175],"B":[0.34885034913271257,2.51564375381825],"C":[0.3328236453418925,2.2003224009629285]},"jevbench_score":1.5079991770819052,"intelligence":{"I_open":17.087609508025615,"I_sealed":5.773666552589584,"base":11.4306380303076,"gap":11.31394295543603,"excess":6.127315455356787,"penalty":1.0,"per_type_split":{"open|choice":{"cc":19.35896876830928,"tiers":{"easy":40.30750678323787,"standard":9.539901221731215,"judge":33.609958506224075,"hard":8.343125734430064},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":6.948716735286226,"tiers":{"easy":17.647058823529417,"standard":6.060606060606055,"judge":45.09803921568627,"hard":-23.893805309734518},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":24.95514302048134,"tiers":{"easy":8.445647112288901,"standard":35.35112046654424,"judge":23.00066710619347,"hard":25.3503852102139},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":8.287901229781768,"tiers":{"easy":0.39062499999998146,"standard":4.912836767036478,"judge":20.49469964664312,"hard":2.7946537059538477},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-13.490909090909096,"tiers":{"easy":-4.0000000000000036,"standard":-16.000000000000004,"judge":-16.000000000000004,"hard":-12.727272727272732},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":22.52400751889608,"tiers":{"easy":24.610220611754297,"standard":27.203564617800758,"judge":19.93825422047939,"hard":21.601990670041705},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":48.48351103482105,"parts":{"choice":{"ece_hard":0.37154133110131926,"ece_score":25.691733779736147,"mean_tvd":0.4915353240795216,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":38.26910068589199},"noul":{"ece":0.2160041611187395,"brier":0.2602310957635408,"n":495.0,"score":56.799167776252105,"abstention_rate":0.296969696969697},"score":{"nrps":0.23115691335442487,"top_ece":0.38059889689959703,"n":330.0,"interval90_coverage":0.7181818181818181,"interval90_width":0.5527777777777781,"score":50.38226464231905}}},"speed":{"p50_s_raw":0.016555813956074417,"p95_s_raw":0.0531386190210469,"p50_s_adjusted":0.18311162791214883,"p95_s_adjusted":0.2562772380420938,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.044662660714285714,"basis":"price floor: base-model reference price applied"},"rank":62,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":62,"B":61,"C":62}},{"key":"kev-0.6b","display":"kev 0.6B (research preview)","addendum":null,"author":"Jared Palmer","repo":"https://github.com/jaredpalmer/kev","class":"jev-rebuild","licence":"Apache-2.0","open":true,"underlying":"Qwen3-0.6B-Base + LoRA + learned pointer head; jaredpalmer/kev-0.6b","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/kev-0.6b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":10.335956592880297,"calibration":67.65469262034678,"speed":87.21666696915125,"cost":80.09400261254245},"scores":{"A":1.2616561727973594,"B":0.920048498843254,"C":0.8761501199981665},"views":{"Balanced 33:33:33":1.0621184537440649,"Emphasis on Accuracy 60:20:20":0.6800256161399014,"Emphasis on Cost 20:20:60":1.4667540546597662,"Emphasis on Speed 20:60:20":1.487578762118429,"Intelligence only":0.4416843604758039},"composite_ci95":{"A":[0.4895664854415589,2.5047442312337607],"B":[0.34458355059432716,1.8834459713121963],"C":[0.33997672600108253,1.739405716134556]},"jevbench_score":1.2616561727973594,"intelligence":{"I_open":25.463748221534246,"I_sealed":-1.491216071552195,"base":11.986266074991025,"gap":26.95496429308644,"excess":21.768336793007197,"penalty":0.862316632069928,"per_type_split":{"open|choice":{"cc":33.250533580633615,"tiers":{"easy":85.52909255351221,"standard":53.21029373537822,"judge":29.460580912863083,"hard":13.043478260869541},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":1.9626772672061623,"tiers":{"easy":41.176470588235304,"standard":-21.212121212121215,"judge":41.176470588235304,"hard":-25.663716814159287},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":41.17803381676297,"tiers":{"easy":37.126562886703404,"standard":56.85277405832456,"judge":39.56389282440696,"hard":35.564137172764056},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":21.314071279675943,"tiers":{"easy":50.1953125,"standard":23.930269413629183,"judge":15.194346289752657,"hard":17.375455650060772},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-57.6,"tiers":{"easy":-43.99999999999999,"standard":-56.00000000000001,"judge":-60.0,"hard":-60.0},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":31.812280505667474,"tiers":{"easy":43.407238368764425,"standard":37.62475155995868,"judge":28.11632890331999,"hard":28.77926921450824},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.003754693366708417},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0060606060606061}},"calibration":{"score":67.65469262034678,"parts":{"choice":{"ece_hard":0.22409167003656888,"ece_score":55.18166599268623,"mean_tvd":0.3545293454678935,"n_tvd":50.0,"n":796.0,"n_ece_hard":276.0,"score":59.864365722948435},"noul":{"ece":0.18842424242424205,"brier":0.2551985858585859,"n":495.0,"score":62.31515151515159,"abstention_rate":0.42828282828282827},"score":{"nrps":0.1609289678519071,"top_ece":0.1116899098446431,"n":328.0,"interval90_coverage":0.948170731707317,"interval90_width":0.7632113821138208,"score":80.78456062294033}}},"speed":{"p50_s_raw":0.13577913912013173,"p95_s_raw":0.15013638343662017,"p50_s_adjusted":0.4215582782402635,"p95_s_adjusted":0.45027276687324036,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.004608220443349753,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":63,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":63,"B":63,"C":63}},{"key":"raw-qwen3-0.6b","display":"Raw Qwen3 0.6B direct logits","addendum":null,"author":"Alibaba Qwen / neutral reproduction","repo":"https://huggingface.co/Qwen/Qwen3-0.6B","class":"raw-logit-control","licence":"Apache-2.0","open":"yes","underlying":"Qwen3-0.6B BF16","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/raw-qwen3-0.6b/spec.json:raw_adapter.RawControlAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":10.561366022298714,"calibration":21.45426398042649,"speed":90.39019014190508,"cost":77.58073177551512},"scores":{"A":1.079997575465549,"B":0.8582394645786061,"C":0.7499983162955202},"views":{"Balanced 33:33:33":1.1282330954187632,"Emphasis on Accuracy 60:20:20":0.7242849176442601,"Emphasis on Cost 20:20:60":1.5447246246138013,"Emphasis on Speed 20:60:20":1.5848154665864036,"Intelligence only":0.47121626612423784},"composite_ci95":{"A":[0.27446277660594287,2.612621391882716],"B":[0.20450823859446796,2.166446961168533],"C":[0.1905991504207937,1.8143204110296636]},"jevbench_score":1.079997575465549,"intelligence":{"I_open":10.0105735585918,"I_sealed":11.112158486005626,"base":10.561366022298714,"gap":-1.1015849274138265,"excess":-6.288212427493069,"penalty":1.0,"per_type_split":{"open|choice":{"cc":17.1202876731998,"tiers":{"easy":80.10250226107928,"standard":31.37509747855471,"judge":0.4149377593361205,"hard":6.776341558950235},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":13.422302146924743,"tiers":{"easy":11.764705882352944,"standard":3.0303030303030276,"judge":47.058823529411775,"hard":-6.194690265486724},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":-0.510869144349142,"tiers":{"easy":-9.384680024557767,"standard":-5.394614995523739,"judge":-16.620508057527793,"hard":16.231685686174302},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":23.798922438502146,"tiers":{"easy":50.1953125,"standard":17.138329182703217,"judge":23.144876325088344,"hard":21.0206561360875},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-4.254545454545457,"tiers":{"easy":-36.0,"standard":-12.0,"judge":-16.000000000000004,"hard":16.36363636363636},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":13.792098474060186,"tiers":{"easy":14.761469685852314,"standard":12.256011452425764,"judge":16.30759642915529,"hard":12.431175715608035},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":21.45426398042649,"parts":{"choice":{"ece_hard":0.5472022432112024,"ece_score":0.0,"mean_tvd":0.5290557394596033,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":23.547213027019836},"noul":{"ece":0.46457443043863456,"brier":0.4635923548486851,"n":495.0,"score":7.085113912273089,"abstention_rate":0.004040404040404066},"score":{"nrps":0.3253906999602692,"top_ece":0.5113738710605674,"n":330.0,"interval90_coverage":0.5363636363636364,"interval90_width":0.3712121212121212,"score":33.730465001986545}}},"speed":{"p50_s_raw":0.06320727849379182,"p95_s_raw":0.09034462417475878,"p50_s_adjusted":0.27641455698758366,"p95_s_adjusted":0.3306892483495175,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.005588676108374384,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":64,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":64,"B":64,"C":64}},{"key":"gliner2.5-small","display":"GLiNER2.5 small (Fastino, 74M)","addendum":null,"author":"Fastino","repo":"https://huggingface.co/fastino/gliner2.5-small-v1","class":"classifier","licence":"Apache-2.0","open":"yes","underlying":"GLiNER2.5 small schema-conditioned extractor, 74M","endpoint_kind":"cpu","endpoint_condition":"evaluator-owned CPU container, offline read-only, pinned cores","gpu":"cpu","adapter_id":"images/gliner2.5-small/spec.json:warm_adapter.WarmLoaded","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":9.18733556299694,"calibration":55.90409648375797,"speed":77.35928845375412,"cost":86.62313527343974},"scores":{"A":0.893172274587778,"B":0.6491615526073343,"C":0.6202585240192903},"views":{"Balanced 33:33:33":0.7597604657587358,"Emphasis on Accuracy 60:20:20":0.4809421780175859,"Emphasis on Cost 20:20:60":1.0793408837724145,"Emphasis on Speed 20:60:20":1.0605921103009057,"Intelligence only":0.31019066833311315},"composite_ci95":{"A":[0.23120097407975104,2.1950672601130257],"B":[0.1602676167296053,1.6603502833457342],"C":[0.1605562319998271,1.5243522639673786]},"jevbench_score":0.893172274587778,"intelligence":{"I_open":17.164169957518137,"I_sealed":1.648634129566223,"base":9.40640204354218,"gap":15.515535827951915,"excess":10.328908327872671,"penalty":0.9767109167212733,"per_type_split":{"open|choice":{"cc":15.71439352604785,"tiers":{"easy":69.24932167621344,"standard":25.136469976605138,"judge":6.639004149377612,"hard":4.426165295730488},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":4.308204375877466,"tiers":{"easy":17.647058823529417,"standard":-24.242424242424242,"judge":47.058823529411775,"hard":-16.814159292035402},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":31.4699119706291,"tiers":{"easy":12.58206671454689,"standard":38.491751312752264,"judge":34.830715051559395,"hard":30.160351302890355},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":11.865639447234665,"tiers":{"easy":23.828124999999982,"standard":13.063165044147636,"judge":4.593639575971737,"hard":13.730255164034043},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-34.10909090909091,"tiers":{"easy":-43.99999999999999,"standard":-52.0,"judge":-28.000000000000004,"hard":-27.27272727272727},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":27.189353850554916,"tiers":{"easy":35.38743812791506,"standard":28.853182750666086,"judge":23.661402666683927,"hard":26.953881719062533},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":55.90409648375797,"parts":{"choice":{"ece_hard":0.14635085613073423,"ece_score":70.72982877385316,"mean_tvd":0.3114951726171251,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":69.79015575607032},"noul":{"ece":0.37137972333206415,"brier":0.3843988929195394,"n":495.0,"score":25.72405533358717,"abstention_rate":0.19191919191919193},"score":{"nrps":0.2018497964611113,"top_ece":0.1770943181532802,"n":330.0,"interval90_coverage":0.8787878787878788,"interval90_width":0.7929292929292928,"score":72.19807836161641}}},"speed":{"p50_s_raw":0.16102993860840797,"p95_s_raw":1.8705574963241733,"p50_s_adjusted":0.47205987721681597,"p95_s_adjusted":3.8911149926483466,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.002791871921182266,"basis":"ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies"},"rank":65,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":65,"B":66,"C":65}},{"key":"raw-qwen3-1.7b","display":"Raw Qwen3 1.7B direct logits","addendum":null,"author":"Alibaba Qwen / neutral reproduction","repo":"https://huggingface.co/Qwen/Qwen3-1.7B","class":"raw-logit-control","licence":"Apache-2.0","open":"yes","underlying":"Qwen3-1.7B BF16","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/raw-qwen3-1.7b/spec.json:raw_adapter.RawControlAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":9.67276350131027,"calibration":21.572869316943088,"speed":90.22368770839446,"cost":68.54983190559568},"scores":{"A":0.8534322366651821,"B":0.6711978961938931,"C":0.5926612754619321},"views":{"Balanced 33:33:33":0.869979501977677,"Emphasis on Accuracy 60:20:20":0.5572161102672888,"Emphasis on Cost 20:20:60":1.1826089384941272,"Emphasis on Speed 20:60:20":1.2374197554240198,"Intelligence only":0.36200260818869395},"composite_ci95":{"A":[0.14697567748045587,2.284188740424065],"B":[0.10677588994673629,1.8948794983135524],"C":[0.10206644269476102,1.5862421808500455]},"jevbench_score":0.8534322366651821,"intelligence":{"I_open":12.266077251349891,"I_sealed":7.079449751270649,"base":9.67276350131027,"gap":5.186627500079243,"excess":0.0,"penalty":1.0,"per_type_split":{"open|choice":{"cc":41.03570373377122,"tiers":{"easy":72.86704853783539,"standard":40.73303873147908,"judge":52.282157676348554,"hard":24.794359576968255},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":11.657596264571797,"tiers":{"easy":5.882352941176472,"standard":3.0303030303030276,"judge":43.13725490196079,"hard":-6.194690265486724},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":-15.895068244293338,"tiers":{"easy":-33.71272549854139,"standard":-16.03158197732668,"judge":-35.13756131391654,"hard":3.059472738002744},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":23.292766488000304,"tiers":{"easy":47.265624999999986,"standard":28.00543355218476,"judge":29.77031802120142,"hard":10.085054678007312},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-3.454545454545456,"tiers":{"easy":-36.0,"standard":-7.9999999999999964,"judge":-16.000000000000004,"hard":16.36363636363636},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":1.4001282203570966,"tiers":{"easy":3.8269954562980213,"standard":12.432023947196736,"judge":3.6923303625473913,"hard":-6.4416880586906755},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":21.572869316943088,"parts":{"choice":{"ece_hard":0.5445983642845623,"ece_score":0.0,"mean_tvd":0.4309651586811289,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":28.451742065943552},"noul":{"ece":0.46974567785208526,"brier":0.4695975444848059,"n":495.0,"score":6.050864429582948,"abstention_rate":0.0},"score":{"nrps":0.39567997089394474,"top_ece":0.6812260889715819,"n":330.0,"interval90_coverage":0.3393939393939394,"interval90_width":0.1431818181818182,"score":30.216001455302766}}},"speed":{"p50_s_raw":0.06600576639175415,"p95_s_raw":0.09339700550772245,"p50_s_adjusted":0.2820115327835083,"p95_s_adjusted":0.3367940110154449,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.011177352216748768,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":66,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":66,"B":65,"C":66}},{"key":"mirror","display":"Mirror","addendum":null,"author":"Bluusun","repo":"https://github.com/Bluusun/Decision-API","class":"jev-rebuild","licence":"Apache-2.0 wrapper and Mirror release; upstream DeBERTa/model assets retain their own terms","open":"yes","underlying":"Mirror DeBERTa-v3-large 436M classification-span scorer","endpoint_kind":"cpu","endpoint_condition":"evaluator-owned CPU container, offline read-only, pinned cores","gpu":"cpu","adapter_id":"images/mirror/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1253},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":5.595966086663697,"calibration":43.210973211362244,"speed":63.64487086148672,"cost":89.27300805786774},"scores":{"A":0.21902679657747523,"B":0.15370891108333642,"C":0.15210194206769112},"views":{"Balanced 33:33:33":0.1827590337457119,"Emphasis on Accuracy 60:20:20":0.11123994452772705,"Emphasis on Cost 20:20:60":0.2746709218182245,"Emphasis on Speed 20:60:20":0.2642175207718493,"Intelligence only":0.07009470509798131},"composite_ci95":{"A":[0.014890703888250692,0.9013602095664797],"B":[0.009792378233239713,0.6688203363147419],"C":[0.010340766589062977,0.6259445899767221]},"jevbench_score":0.21902679657747523,"intelligence":{"I_open":6.74523751875524,"I_sealed":4.4466946545721555,"base":5.595966086663697,"gap":2.2985428641830845,"excess":-2.888084635896158,"penalty":1.0,"per_type_split":{"open|choice":{"cc":1.6362965626667982,"tiers":{"easy":69.24932167621344,"standard":23.576813101117754,"judge":-3.7344398340248746,"hard":-22.209165687426584},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-0.6142634044768318,"tiers":{"easy":47.058823529411775,"standard":0.0,"judge":11.764705882352944,"hard":-22.123893805309734},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":19.213679398075755,"tiers":{"easy":-3.1763500509754294,"standard":27.541232473948863,"judge":34.02986858621417,"hard":9.535268331298186},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":-9.371752896132213,"tiers":{"easy":6.249999999999979,"standard":4.912836767036478,"judge":-4.681978798586563,"hard":-23.936816524908842},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-4.545454545454547,"tiers":{"easy":19.999999999999996,"standard":-12.0,"judge":8.000000000000007,"hard":-16.36363636363637},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":27.257291405303228,"tiers":{"easy":38.978075851334324,"standard":26.205089036101235,"judge":26.98713233165959,"hard":25.055815783629175},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.39424280350438046},"noul":{"n":495,"invalid_rate":0.09696969696969693},"score":{"n":330,"invalid_rate":0.024242424242424288}},"calibration":{"score":43.210973211362244,"parts":{"choice":{"ece_hard":0.3664447619629182,"ece_score":26.71104760741636,"mean_tvd":0.4119599465354314,"n_tvd":26.0,"n":484.0,"n_ece_hard":68.0,"score":42.75752647693662},"noul":{"ece":0.40004330431713914,"brier":0.40198105556852537,"n":447.0,"score":19.991339136572172,"abstention_rate":0.046979865771812124},"score":{"nrps":0.20196336767408615,"top_ece":0.2301777759571773,"n":322.0,"interval90_coverage":0.8913043478260869,"interval90_width":0.8027950310559004,"score":66.88405402057796}}},"speed":{"p50_s_raw":2.2142462366027758,"p95_s_raw":4.643026909977197,"p50_s_adjusted":4.578492473205552,"p95_s_adjusted":9.436053819954394,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.0022780665024630543,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":67,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":67,"B":67,"C":67}},{"key":"zerank-2","display":"ZeroEntropy zerank-2","addendum":null,"author":"ZeroEntropy","repo":"https://huggingface.co/zeroentropy/zerank-2-reranker","class":"reranker","licence":"Apache-2.0","open":"yes","underlying":"Qwen3-Reranker-derived 4B cross-encoder","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/zerank-2/spec.json:reranker_adapter.RerankerAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":4.764817160947821,"calibration":81.70676532515279,"speed":80.27703860657243,"cost":48.39770052432463},"scores":{"A":0.13334937595297605,"B":0.09147122359399754,"C":0.0926037333006778},"views":{"Balanced 33:33:33":0.10504941162244104,"Emphasis on Accuracy 60:20:20":0.06419374610133761,"Emphasis on Cost 20:20:60":0.14963468464765028,"Emphasis on Speed 20:60:20":0.15880059700680335,"Intelligence only":0.04054227910564448},"composite_ci95":{"A":[0.012109200548825812,0.48542181877192503],"B":[0.007905764526842694,0.3479463361796913],"C":[0.008409167047795702,0.33709848525828134]},"jevbench_score":0.13334937595297605,"intelligence":{"I_open":9.967956631928743,"I_sealed":-0.43832231003310085,"base":4.764817160947821,"gap":10.406278941961844,"excess":5.219651441882601,"penalty":1.0,"per_type_split":{"open|choice":{"cc":56.57069726813885,"tiers":{"easy":90.95568284594513,"standard":65.68754873927736,"judge":70.95435684647303,"hard":32.6282804543674},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-68.06887195747166,"tiers":{"easy":-41.17647058823529,"standard":-78.78787878787878,"judge":-74.50980392156863,"hard":-64.60176991150442},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":41.402044585119036,"tiers":{"easy":8.991045588911728,"standard":45.39451542604084,"judge":41.19594702284447,"hard":47.66313208541588},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":46.961454410449676,"tiers":{"easy":64.84374999999999,"standard":45.66447815259227,"judge":45.67137809187279,"hard":44.10692588092346},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-85.63636363636364,"tiers":{"easy":-76.0,"standard":-92.0,"judge":-80.0,"hard":-89.0909090909091},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":37.35994229581467,"tiers":{"easy":48.814492412237634,"standard":41.111792494891034,"judge":35.220481360549726,"hard":34.22497536861945},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":81.70676532515279,"parts":{"choice":{"ece_hard":0.09639665979788783,"ece_score":80.72066804042244,"mean_tvd":0.26735237395866535,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":76.99271532227795},"noul":{"ece":0.08297814786406878,"brier":0.21755062612673598,"n":495.0,"score":83.40437042718625,"abstention_rate":0.8363636363636364},"score":{"nrps":0.15146723973315543,"top_ece":0.07703427787348029,"n":330.0,"interval90_coverage":0.9515151515151515,"interval90_width":0.8166666666666661,"score":84.7232102259942}}},"speed":{"p50_s_raw":0.16138588916510344,"p95_s_raw":0.9172351316083216,"p50_s_adjusted":0.4727717783302069,"p95_s_adjusted":1.984470263216643,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.05249000923645321,"basis":"ESTIMATE: size-class proxy (deepinfra:Qwen/Qwen3-Reranker-4B); no exact base-model floor applies"},"rank":68,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":68,"B":68,"C":68}},{"key":"jeff","display":"jeff (Logan Markewich, GLiFormer 400M)","addendum":null,"author":"Logan Markewich","repo":"https://github.com/logan-markewich/jeff","class":"jev-rebuild","licence":"MIT (code); GLiFormer weights per their model card","open":"yes","underlying":"GLiFormer large (knowledgator/gliformer-large-v1, ~400M) behind a TypeSafe-compatible /v1/systemone server","endpoint_kind":"cpu","endpoint_condition":"evaluator-owned CPU container, offline read-only, pinned cores","gpu":"cpu","adapter_id":"images/jeff/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1620},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":4.434526796394629,"calibration":80.26674639624511,"speed":55.762758324634696,"cost":81.08682556098348},"scores":{"A":0.11730368929731415,"B":0.07965896456941576,"C":0.08146089534535705},"views":{"Balanced 33:33:33":0.09226315098361863,"Emphasis on Accuracy 60:20:20":0.0556471938321315,"Emphasis on Cost 20:20:60":0.14024725289968296,"Emphasis on Speed 20:60:20":0.13486054271524178,"Intelligence only":0.034882037283846826},"composite_ci95":{"A":[0.003040184337069344,0.5116303479621337],"B":[0.0019501985520175905,0.36371663996890347],"C":[0.0021112391229648223,0.35529885275148176]},"jevbench_score":0.11730368929731415,"intelligence":{"I_open":12.711903221404976,"I_sealed":-3.5604602603254047,"base":4.5757214805397854,"gap":16.27236348173038,"excess":11.085735981651137,"penalty":0.9691426401834886,"per_type_split":{"open|choice":{"cc":33.014740188512285,"tiers":{"easy":80.10250226107928,"standard":48.53132310891603,"judge":34.64730290456433,"hard":12.260086173129631},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-28.086381777167823,"tiers":{"easy":41.176470588235304,"standard":-24.242424242424242,"judge":-21.568627450980394,"hard":-52.212389380530965},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":33.20735125287047,"tiers":{"easy":19.455069124423964,"standard":47.54514134275618,"judge":33.55851900393184,"hard":29.213150926743193},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":21.788319457161514,"tiers":{"easy":35.54687499999999,"standard":32.08059769074034,"judge":17.84452296819789,"hard":16.160388821385194},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-60.763636363636365,"tiers":{"easy":-36.0,"standard":-60.0,"judge":-56.00000000000001,"hard":-70.9090909090909},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":28.29393612549864,"tiers":{"easy":30.703170289855073,"standard":31.066624685138567,"judge":24.574894692502124,"hard":29.09456437933695},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0025031289111389077},"noul":{"n":495,"invalid_rate":0.004040404040404066},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":80.26674639624511,"parts":{"choice":{"ece_hard":0.10291963636363644,"ece_score":79.4160727272727,"mean_tvd":0.32331250001667494,"n_tvd":50.0,"n":797.0,"n_ece_hard":275.0,"score":73.5424113628026},"noul":{"ece":0.10606815415821512,"brier":0.2296393906288032,"n":493.0,"score":78.78636916835697,"abstention_rate":0.5841784989858012},"score":{"nrps":0.17250476624242422,"top_ece":0.02903303030303039,"n":330.0,"interval90_coverage":0.9878787878787879,"interval90_width":0.9186868686868686,"score":88.47145865757575}}},"speed":{"p50_s_raw":3.4936436004936695,"p95_s_raw":18.50993296541273,"p50_s_adjusted":7.137287200987339,"p95_s_adjusted":37.16986593082546,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.004270110837438424,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":69,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":69,"B":69,"C":69}},{"key":"smalljev","display":"smalljev semantic-v9","addendum":null,"author":"Aditya (isHeSatoshi)","repo":"https://github.com/isHeSatoshi/smalljev","class":"jev-rebuild","licence":"Apache-2.0","open":"yes","underlying":"MiniCPM5-2B-Base, 2.5B dense, with LoRA and native semantic decision heads","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/smalljev/spec.json:remote_inproc_v15.RemoteInprocV15Adapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":3.659593452439375,"calibration":73.08838344851645,"speed":86.46386328362564,"cost":60.77469204714053},"scores":{"A":0.06803548717120994,"B":0.04553683400675546,"C":0.047246866091118014},"views":{"Balanced 33:33:33":0.05334393826915162,"Emphasis on Accuracy 60:20:20":0.031594465353347985,"Emphasis on Cost 20:20:60":0.08015154153596153,"Emphasis on Speed 20:60:20":0.08256726252157968,"Intelligence only":0.019604623987683183},"composite_ci95":{"A":[0.00020953618518935847,0.37562154070461956],"B":[0.00013237047164362027,0.2637008130971111],"C":[0.0001455112397148323,0.2608482921559858]},"jevbench_score":0.06803548717120994,"intelligence":{"I_open":10.933407092326295,"I_sealed":-3.5202907865888893,"base":3.706558152868703,"gap":14.453697878915184,"excess":9.267070378835943,"penalty":0.9873292962116406,"per_type_split":{"open|choice":{"cc":31.14027036673812,"tiers":{"easy":80.10250226107928,"standard":37.61372498050429,"judge":36.721991701244825,"hard":11.476694085389722},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-40.403672329752496,"tiers":{"easy":41.176470588235304,"standard":-18.181818181818176,"judge":-66.66666666666667,"hard":-52.212389380530965},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":42.06362323999326,"tiers":{"easy":8.729240242477642,"standard":50.205876167206775,"judge":42.3552220235646,"hard":46.107393438086916},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":18.27162475771627,"tiers":{"easy":38.47656249999998,"standard":11.70477699796244,"judge":13.869257950530042,"hard":19.805589307411925},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-63.81818181818182,"tiers":{"easy":-36.0,"standard":-43.99999999999999,"judge":-72.0,"hard":-74.54545454545455},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":34.98568470069888,"tiers":{"easy":51.90848980395907,"standard":38.71916059833131,"judge":35.11399183902504,"hard":28.792015122323},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":73.08838344851645,"parts":{"choice":{"ece_hard":0.1714385460645283,"ece_score":65.71229078709435,"mean_tvd":0.38015266094584205,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":63.84851234625506},"noul":{"ece":0.15899537362210042,"brier":0.24502031712995248,"n":495.0,"score":68.20092527557992,"abstention_rate":0.6646464646464647},"score":{"nrps":0.1554815062367462,"top_ece":0.05010211964448292,"n":330.0,"interval90_coverage":0.9606060606060606,"interval90_width":0.8315656565656561,"score":87.2157127237144}}},"speed":{"p50_s_raw":0.15031232591718435,"p95_s_raw":0.17547750547528257,"p50_s_adjusted":0.45062465183436873,"p95_s_adjusted":0.5009550109505652,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.020300665024630543,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":70,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":70,"B":70,"C":70}},{"key":"opendecision","display":"OpenDecision (ModernBERT-large zero-shot)","addendum":null,"author":"Deepan Wadhwa","repo":"https://github.com/deepanwadhwa/OpenDecision","class":"classifier","licence":"Apache-2.0","open":"yes","underlying":"MoritzLaurer/ModernBERT-large-zeroshot-v2.0 (~400M) used as a zero-shot NLI decision engine","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/opendecision/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":2.0712188139108285,"calibration":72.52157722321965,"speed":86.5838203210334,"cost":79.10704862374335},"scores":{"A":0.013179888010630534,"B":0.008549163739426961,"C":0.009152700007382313},"views":{"Balanced 33:33:33":0.010153759421615222,"Emphasis on Accuracy 60:20:20":0.005826306011620282,"Emphasis on Cost 20:20:60":0.01611912861066923,"Emphasis on Speed 20:60:20":0.01618551488693281,"Intelligence only":0.00355416788559629},"composite_ci95":{"A":[0.0,0.19294313592439996],"B":[0.0,0.13147202005412764],"C":[0.0,0.13398828883638889]},"jevbench_score":0.013179888010630534,"intelligence":{"I_open":9.469704530612013,"I_sealed":-5.262248023254946,"base":2.1037282536785336,"gap":14.73195255386696,"excess":9.545325053787717,"penalty":0.9845467494621228,"per_type_split":{"open|choice":{"cc":34.171161143191846,"tiers":{"easy":85.52909255351221,"standard":43.85235248245385,"judge":42.946058091286325,"hard":9.909909909909892},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-47.144006436041835,"tiers":{"easy":-5.882352941176472,"standard":-45.45454545454546,"judge":-64.70588235294117,"hard":-45.13274336283186},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":41.38195888468603,"tiers":{"easy":4.952451296489713,"standard":48.63539699723373,"judge":45.60107321577297,"hard":43.698280977146055},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":20.037419907604885,"tiers":{"easy":35.54687499999999,"standard":17.138329182703217,"judge":13.869257950530042,"hard":22.235722964763077},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-70.9090909090909,"tiers":{"easy":-52.0,"standard":-92.0,"judge":-68.0,"hard":-67.27272727272727},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":35.08492693172118,"tiers":{"easy":42.30553527502682,"standard":34.52664950313588,"judge":33.423748326181055,"hard":34.80479751434251},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":72.52157722321965,"parts":{"choice":{"ece_hard":0.15882113167094838,"ece_score":68.23577366581033,"mean_tvd":0.3451301543433275,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":66.8613791157388},"noul":{"ece":0.14797539912879165,"brier":0.2628657210726989,"n":495.0,"score":70.40492017424167,"abstention_rate":0.6646464646464647},"score":{"nrps":0.16442792259044475,"top_ece":0.11480171490799297,"n":330.0,"interval90_coverage":0.9515151515151515,"interval90_width":0.747474747474747,"score":80.29843237967847}}},"speed":{"p50_s_raw":0.07342815498122945,"p95_s_raw":0.29486368095385834,"p50_s_adjusted":0.29685630996245893,"p95_s_adjusted":0.7397273619077167,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.004970862068965518,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":71,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":71,"B":71,"C":71}},{"key":"bge-reranker-v2-m3","display":"BAAI bge-reranker-v2-m3","addendum":null,"author":"BAAI","repo":"https://huggingface.co/BAAI/bge-reranker-v2-m3","class":"reranker","licence":"Apache-2.0","open":"yes","underlying":"568M multilingual cross-encoder","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/bge-reranker-v2-m3/spec.json:reranker_adapter.RerankerAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":0.0,"calibration":83.32638523205513,"speed":90.54571136520154,"cost":59.325605115070715},"scores":{"A":0.0,"B":0.0,"C":0.0},"views":{"Balanced 33:33:33":0.0,"Emphasis on Accuracy 60:20:20":0.0,"Emphasis on Cost 20:20:60":0.0,"Emphasis on Speed 20:60:20":0.0,"Intelligence only":0.0},"composite_ci95":{"A":[0.0,0.0],"B":[0.0,0.0],"C":[0.0,0.0]},"jevbench_score":0.0,"intelligence":{"I_open":-21.098300472845224,"I_sealed":-22.804972321036335,"base":-21.951636396940778,"gap":1.7066718481911103,"excess":-3.4799556518881323,"penalty":1.0,"per_type_split":{"open|choice":{"cc":4.407461395812141,"tiers":{"easy":18.601145613506187,"standard":9.539901221731215,"judge":1.452282157676369,"hard":0.5092048570309202},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-100.0,"tiers":{"easy":-100.0,"standard":-100.0,"judge":-100.0,"hard":-100.0},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":32.29763718565218,"tiers":{"easy":7.168115899391115,"standard":33.2624979548925,"judge":39.87814650710333,"hard":32.41220513150892},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":4.202706795987319,"tiers":{"easy":23.828124999999982,"standard":8.98800090559206,"judge":-2.0318021201413345,"hard":1.5795868772782722},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-100.0,"tiers":{"easy":-100.0,"standard":-100.0,"judge":-100.0,"hard":-100.0},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":27.382376240903668,"tiers":{"easy":39.25333793379111,"standard":27.66412381361546,"judge":26.017247074792348,"hard":25.297608905909396},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":83.32638523205513,"parts":{"choice":{"ece_hard":0.03376010460233937,"ece_score":93.24797907953213,"mean_tvd":0.33300482957646693,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":79.97374806094271},"noul":{"ece":0.08650888354314805,"brier":0.2560880956241559,"n":495.0,"score":82.69822329137038,"abstention_rate":1.0},"score":{"nrps":0.18556025788685246,"top_ece":0.03414802761805064,"n":330.0,"interval90_coverage":1.0,"interval90_width":1.0,"score":87.30718434385231}}},"speed":{"p50_s_raw":0.02942055370658636,"p95_s_raw":0.1361460106447339,"p50_s_adjusted":0.20884110741317272,"p95_s_adjusted":0.42229202128946774,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.022688885467980296,"basis":"ESTIMATE: base-model market reference (deepinfra:BAAI/bge-m3)"},"rank":72,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":72,"B":72,"C":72}},{"key":"certo","display":"Certo v1 (AltSlate Labs)","addendum":null,"author":"AltSlate Labs","repo":"https://huggingface.co/altslate/certo-decision-model","class":"jev-rebuild","licence":"MIT","open":"yes","underlying":"ModernBERT-large with a per-option query/scoring head, ~400M parameters","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/certo/spec.json:certo_v15.CertoLocalV15Adapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":0.0,"calibration":87.52088082714991,"speed":91.42520786881005,"cost":96.9130801359906},"scores":{"A":0.0,"B":0.0,"C":0.0},"views":{"Balanced 33:33:33":0.0,"Emphasis on Accuracy 60:20:20":0.0,"Emphasis on Cost 20:20:60":0.0,"Emphasis on Speed 20:60:20":0.0,"Intelligence only":0.0},"composite_ci95":{"A":[0.0,0.0],"B":[0.0,0.0],"C":[0.0,0.0]},"jevbench_score":0.0,"intelligence":{"I_open":-24.120274249716694,"I_sealed":-24.930324931123515,"base":-24.525299590420104,"gap":0.8100506814068211,"excess":-4.376576818672421,"penalty":1.0,"per_type_split":{"open|choice":{"cc":-1.8459740781458573,"tiers":{"easy":5.939101597829376,"standard":-2.9373537821679307,"judge":-5.809128630705371,"hard":-0.2741872307089941},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-100.0,"tiers":{"easy":-100.0,"standard":-100.0,"judge":-100.0,"hard":-100.0},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":29.48515132899575,"tiers":{"easy":11.787672811059903,"standard":32.471201413427586,"judge":37.48872870249018,"hard":26.413812886142974},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":-2.0457518184464063,"tiers":{"easy":-8.39843750000002,"standard":-4.595879556259874,"judge":-4.681978798586563,"hard":2.7946537059538477},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-100.0,"tiers":{"easy":-100.0,"standard":-100.0,"judge":-100.0,"hard":-100.0},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":27.254777025075853,"tiers":{"easy":38.20869565217393,"standard":26.696263643996645,"judge":25.918702611625953,"hard":25.79760986892836},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":87.52088082714991,"parts":{"choice":{"ece_hard":0.029833574007220198,"ece_score":94.03328519855596,"mean_tvd":0.3310568334499983,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":80.46380092677806},"noul":{"ece":0.034504242424242486,"brier":0.24896424921212137,"n":495.0,"score":93.0991515151515,"abstention_rate":1.0},"score":{"nrps":0.18789589617929317,"top_ece":0.01605515151515168,"n":330.0,"interval90_coverage":1.0,"interval90_width":0.9944444444444445,"score":88.99969003952017}}},"speed":{"p50_s_raw":0.05686726560816169,"p95_s_raw":0.061547018261626354,"p50_s_adjusted":0.2637345312163234,"p95_s_adjusted":0.2730940365232527,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.0012673522167487683,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":73,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":73,"B":73,"C":73}},{"key":"decision-fast","display":"Decision Fast (FlyMy.AI, v53a)","addendum":null,"author":"FlyMy.AI (@denti)","repo":"https://huggingface.co/flymy-ai/decision-fast-preview","class":"jev-rebuild","licence":"Apache-2.0 notices on the included code and the pinned base; the weights are an evaluation preview under EVALUATION-PERMISSION.md, not a cleared commercial release","open":"yes","underlying":"Qwen/Qwen3-0.6B-Base with a trained LoRA adapter and pointer head (10.6M trainable parameters)","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX6000), offline read-only container","gpu":"RTX6000","adapter_id":"images/decision-fast/spec.json:flymy_v15.FlyMyPackage","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":0.0,"calibration":76.07766497651991,"speed":91.44048940449588,"cost":80.08651867745036},"scores":{"A":0.0,"B":0.0,"C":0.0},"views":{"Balanced 33:33:33":0.0,"Emphasis on Accuracy 60:20:20":0.0,"Emphasis on Cost 20:20:60":0.0,"Emphasis on Speed 20:60:20":0.0,"Intelligence only":0.0},"composite_ci95":{"A":[0.0,0.00015849947083770386],"B":[0.0,9.990132864763427e-05],"C":[0.0,0.00011006907697062771]},"jevbench_score":0.0,"intelligence":{"I_open":7.310364004745869,"I_sealed":-11.695325345216284,"base":-2.1924806702352075,"gap":19.005689349962154,"excess":13.81906184988291,"penalty":0.9418093815011709,"per_type_split":{"open|choice":{"cc":38.3565246934954,"tiers":{"easy":83.72022912270124,"standard":54.76995061086561,"judge":48.13278008298756,"hard":11.476694085389722},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-59.06535421892006,"tiers":{"easy":-11.764705882352944,"standard":-81.81818181818181,"judge":-9.80392156862745,"hard":-96.46017699115043},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":42.63992153966227,"tiers":{"easy":9.858070139613762,"standard":59.88463036349702,"judge":46.091382095862066,"hard":39.624434560607156},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":18.49353134211178,"tiers":{"easy":44.335937499999986,"standard":28.00543355218476,"judge":24.46996466431096,"hard":2.7946537059538477},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-86.83636363636364,"tiers":{"easy":-76.0,"standard":-80.0,"judge":-92.0,"hard":-89.0909090909091},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":33.25685625860301,"tiers":{"easy":46.23191862887507,"standard":41.0924552565227,"judge":25.409301485791623,"hard":31.980957246683673},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":76.07766497651991,"parts":{"choice":{"ece_hard":0.19382906987142728,"ece_score":61.23418602571454,"mean_tvd":0.29804180436481853,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":65.71500279461634},"noul":{"ece":0.09637970835882076,"brier":0.22661488853990605,"n":495.0,"score":80.72405832823584,"abstention_rate":0.8080808080808081},"score":{"nrps":0.16085481262204096,"top_ece":0.10163325562190413,"n":330.0,"interval90_coverage":0.9606060606060606,"interval90_width":0.7941919191919187,"score":81.79393380670754}}},"speed":{"p50_s_raw":0.058117716456763446,"p95_s_raw":0.05978923523798585,"p50_s_adjusted":0.2662354329135269,"p95_s_adjusted":0.2695784704759717,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.004610868226600985,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":74,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":74,"B":74,"C":74}},{"key":"gte-reranker-modernbert-base","display":"Alibaba GTE Reranker ModernBERT-base","addendum":null,"author":"Alibaba-NLP","repo":"https://huggingface.co/Alibaba-NLP/gte-reranker-modernbert-base","class":"reranker","licence":"Apache-2.0","open":"yes","underlying":"149M ModernBERT cross-encoder","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/gte-reranker-modernbert-base/spec.json:reranker_adapter.RerankerAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":0.0,"calibration":75.19518902157515,"speed":91.48515951011706,"cost":69.26965431492843},"scores":{"A":0.0,"B":0.0,"C":0.0},"views":{"Balanced 33:33:33":0.0,"Emphasis on Accuracy 60:20:20":0.0,"Emphasis on Cost 20:20:60":0.0,"Emphasis on Speed 20:60:20":0.0,"Intelligence only":0.0},"composite_ci95":{"A":[0.0,0.0],"B":[0.0,0.0],"C":[0.0,0.0]},"jevbench_score":0.0,"intelligence":{"I_open":-18.627164187197426,"I_sealed":-23.156535387423837,"base":-20.891849787310633,"gap":4.529371200226411,"excess":-0.6572562998528317,"penalty":1.0,"per_type_split":{"open|choice":{"cc":13.023655886527008,"tiers":{"easy":11.365691890262292,"standard":17.33818559916818,"judge":27.385892116182585,"hard":0.5092048570309202},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-100.0,"tiers":{"easy":-100.0,"standard":-100.0,"judge":-100.0,"hard":-100.0},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":31.09485155188071,"tiers":{"easy":7.263813138813424,"standard":30.66862041644295,"judge":38.1165550700604,"hard":31.999449084231635},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":2.3457082085803536,"tiers":{"easy":-11.328125000000023,"standard":4.912836767036478,"judge":4.593639575971737,"hard":2.7946537059538477},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-100.0,"tiers":{"easy":-100.0,"standard":-100.0,"judge":-100.0,"hard":-100.0},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":28.184685629148124,"tiers":{"easy":38.44342935494295,"standard":28.268568534260474,"judge":27.431807026686517,"hard":26.14271719698945},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":75.19518902157515,"parts":{"choice":{"ece_hard":0.054310813389460755,"ece_score":89.13783732210786,"mean_tvd":0.32663092997297427,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":78.23737216240522},"noul":{"ece":0.19477931930463183,"brier":0.29107023790736486,"n":495.0,"score":61.044136139073636,"abstention_rate":1.0},"score":{"nrps":0.18591656289063668,"top_ece":0.044001130922215366,"n":330.0,"interval90_coverage":1.0,"interval90_width":1.0,"score":86.30405876324662}}},"speed":{"p50_s_raw":0.030790242832154036,"p95_s_raw":0.09287206921726458,"p50_s_adjusted":0.21158048566430807,"p95_s_adjusted":0.33574413843452916,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.010576570197044334,"basis":"ESTIMATE: size-class proxy (deepinfra:thenlper/gte-base); no exact base-model floor applies"},"rank":75,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":75,"B":75,"C":75}},{"key":"kev-0.5b","display":"kev 0.5B","addendum":null,"author":"Jared Palmer","repo":"https://github.com/jaredpalmer/kev","class":"jev-rebuild","licence":"Apache-2.0","open":true,"underlying":"Qwen2.5-0.5B + LoRA + learned pointer head; jaredpalmer/kev-0.5b","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/kev-0.5b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":0.0,"calibration":64.98295390603997,"speed":88.45344742064806,"cost":80.09398520305423},"scores":{"A":0.0,"B":0.0,"C":0.0},"views":{"Balanced 33:33:33":0.0,"Emphasis on Accuracy 60:20:20":0.0,"Emphasis on Cost 20:20:60":0.0,"Emphasis on Speed 20:60:20":0.0,"Intelligence only":0.0},"composite_ci95":{"A":[0.0,0.0],"B":[0.0,0.0],"C":[0.0,0.0]},"jevbench_score":0.0,"intelligence":{"I_open":3.443416406201637,"I_sealed":-22.2626245077547,"base":-9.409604050776533,"gap":25.706040913956336,"excess":20.519413413877093,"penalty":0.8748058658612291,"per_type_split":{"open|choice":{"cc":25.23699856419025,"tiers":{"easy":83.72022912270124,"standard":28.255783727579928,"judge":39.834024896265575,"hard":-1.8409714061888192},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-48.02344107393561,"tiers":{"easy":-29.411764705882348,"standard":-87.87878787878788,"judge":15.686274509803933,"hard":-80.53097345132744},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":33.11669172835026,"tiers":{"easy":27.747986780244837,"standard":51.74438424407812,"judge":36.650781722995376,"hard":22.494454211528847},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":11.194986272433392,"tiers":{"easy":23.828124999999982,"standard":11.70477699796244,"judge":17.84452296819789,"hard":2.7946537059538477},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-94.54545454545455,"tiers":{"easy":-100.0,"standard":-92.0,"judge":-92.0,"hard":-96.36363636363636},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":16.562594749757046,"tiers":{"easy":36.13676041517198,"standard":15.847928201720284,"judge":11.217165945743702,"hard":16.035458210431706},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.006257822277847325},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.00303030303030305}},"calibration":{"score":64.98295390603997,"parts":{"choice":{"ece_hard":0.3040174962950841,"ece_score":39.196500740983176,"mean_tvd":0.4046701342800813,"n_tvd":50.0,"n":794.0,"n_ece_hard":275.0,"score":49.36474365648752},"noul":{"ece":0.10218181818181817,"brier":0.23746343434343445,"n":495.0,"score":79.56363636363636,"abstention_rate":0.7656565656565657},"score":{"nrps":0.20914361923773894,"top_ece":0.23522337340116994,"n":329.0,"interval90_coverage":0.8723404255319149,"interval90_width":0.7150455927051664,"score":66.02048169799606}}},"speed":{"p50_s_raw":0.10358892846852541,"p95_s_raw":0.12486683493480086,"p50_s_adjusted":0.35717785693705084,"p95_s_adjusted":0.3997336698696017,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.004608226600985222,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":76,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":76,"B":76,"C":76}},{"key":"laya","display":"Laya (Convai Innovations, ModernBERT-large 421M)","addendum":null,"author":"Convai Innovations","repo":"https://huggingface.co/convaiinnovations/laya","class":"jev-rebuild","licence":"Apache-2.0","open":"yes","underlying":"ModernBERT-large encoder + option-marker decision head, 421M, RLCD-trained","endpoint_kind":"cpu","endpoint_condition":"evaluator-owned CPU container, offline read-only, pinned cores","gpu":"cpu","adapter_id":"images/laya/spec.json:warm_adapter.WarmLoaded","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":0.0,"calibration":73.70996996254986,"speed":73.89376469233966,"cost":84.89402784895724},"scores":{"A":0.0,"B":0.0,"C":0.0},"views":{"Balanced 33:33:33":0.0,"Emphasis on Accuracy 60:20:20":0.0,"Emphasis on Cost 20:20:60":0.0,"Emphasis on Speed 20:60:20":0.0,"Intelligence only":0.0},"composite_ci95":{"A":[0.0,0.0],"B":[0.0,0.0],"C":[0.0,0.0]},"jevbench_score":0.0,"intelligence":{"I_open":-0.47796953967189104,"I_sealed":-13.186035258388955,"base":-6.832002399030423,"gap":12.708065718717064,"excess":7.5214382186378215,"penalty":1.0,"per_type_split":{"open|choice":{"cc":37.096544852752906,"tiers":{"easy":83.72022912270124,"standard":40.73303873147908,"judge":57.4688796680498,"hard":8.343125734430064},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-75.73675326928841,"tiers":{"easy":-41.17647058823529,"standard":-78.78787878787878,"judge":-76.47058823529412,"hard":-82.30088495575221},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":37.20629979751983,"tiers":{"easy":14.670391705069118,"standard":50.6155477031802,"judge":36.02994757536041,"hard":37.0179170344219},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":12.772232378428598,"tiers":{"easy":41.406249999999986,"standard":19.8551052750736,"judge":8.568904593639585,"hard":5.224787363305005},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-81.52727272727273,"tiers":{"easy":-84.0,"standard":-76.0,"judge":-84.0,"hard":-81.81818181818181},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":29.19693457367726,"tiers":{"easy":41.28777173913043,"standard":36.50554156171282,"judge":21.725989890480236,"hard":28.123130300693955},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":73.70996996254986,"parts":{"choice":{"ece_hard":0.14936750902527077,"ece_score":70.12649819494585,"mean_tvd":0.2982611665500017,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":70.15019076997285},"noul":{"ece":0.11330686868686872,"brier":0.2578809390101011,"n":495.0,"score":77.33862626262625,"abstention_rate":0.8303030303030303},"score":{"nrps":0.17930905198989894,"top_ece":0.17393454545454545,"n":330.0,"interval90_coverage":0.8424242424242424,"interval90_width":0.6704545454545449,"score":73.6410928550505}}},"speed":{"p50_s_raw":0.6676969870459288,"p95_s_raw":1.2982571767410263,"p50_s_adjusted":1.4853939740918576,"p95_s_adjusted":2.7465143534820524,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.0031881034482758625,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":77,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":77,"B":77,"C":77}},{"key":"lev-350m","display":"lev-350m (Franck Verrot, LFM2.5-350M)","addendum":null,"author":"Franck Verrot (franckverrot)","repo":"https://github.com/franckverrot/lev","class":"jev-rebuild","licence":"Apache-2.0 (code); weights under LiquidAI's LFM1.0 licence, following the LFM2.5-350M base","open":"yes","underlying":"LiquidAI/LFM2.5-350M with a LoRA and a 6.5M-parameter pointer head (a kev clone)","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX6000), offline read-only container","gpu":"RTX6000","adapter_id":"images/lev-350m/spec.json:lev_v15.LevAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":0.0,"calibration":77.67886674901663,"speed":93.63413423056228,"cost":80.03797760430007},"scores":{"A":0.0,"B":0.0,"C":0.0},"views":{"Balanced 33:33:33":0.0,"Emphasis on Accuracy 60:20:20":0.0,"Emphasis on Cost 20:20:60":0.0,"Emphasis on Speed 20:60:20":0.0,"Intelligence only":0.0},"composite_ci95":{"A":[0.0,0.0],"B":[0.0,0.0],"C":[0.0,0.0]},"jevbench_score":0.0,"intelligence":{"I_open":5.528286983062985,"I_sealed":-18.014062268821647,"base":-6.242887642879331,"gap":23.54234925188463,"excess":18.355721751805387,"penalty":0.8964427824819461,"per_type_split":{"open|choice":{"cc":33.06242798398311,"tiers":{"easy":83.72022912270124,"standard":46.97166623342864,"judge":35.68464730290457,"hard":11.476694085389722},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-52.199927436783234,"tiers":{"easy":-41.17647058823529,"standard":-75.75757575757575,"judge":11.764705882352944,"hard":-91.1504424778761},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":35.72236040198907,"tiers":{"easy":10.21198156682026,"standard":48.57199646643111,"judge":37.38335517693317,"hard":34.429390997352186},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":15.013955037546538,"tiers":{"easy":35.54687499999999,"standard":19.8551052750736,"judge":9.8939929328622,"hard":11.300121506682888},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-100.0,"tiers":{"easy":-100.0,"standard":-100.0,"judge":-100.0,"hard":-100.0},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":30.943858155988508,"tiers":{"easy":50.182971014492765,"standard":30.069689336691873,"judge":31.40985678180286,"hard":26.221665381649995},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":77.67886674901663,"parts":{"choice":{"ece_hard":0.07782310469314083,"ece_score":84.43537906137183,"mean_tvd":0.33867733339999334,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":75.28382286068626},"noul":{"ece":0.12583434343434344,"brier":0.23619623030303005,"n":495.0,"score":74.83313131313132,"abstention_rate":0.8},"score":{"nrps":0.17304344217171722,"top_ece":0.0842818181818181,"n":330.0,"interval90_coverage":0.9878787878787879,"interval90_width":0.941666666666667,"score":82.91964607323233}}},"speed":{"p50_s_raw":0.02207189949695021,"p95_s_raw":0.03654062089044591,"p50_s_adjusted":0.1941437989939004,"p95_s_adjusted":0.2230812417808918,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.00462807881773399,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":78,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":78,"B":78,"C":78}},{"key":"mghafiri-qwen3.5-0.8b-decision-model","display":"Qwen3.5-0.8B Decision Model (Mourad Ghafiri)","addendum":null,"author":"Mourad Ghafiri","repo":"https://huggingface.co/mghafiri/qwen3.5-0.8B-decision-model","class":"jev-rebuild","licence":"MIT code and training data; Apache-2.0 model weights","open":"yes","underlying":"Qwen3.5-0.8B-Base fine-tuned as a JevLite decision model with per-question calibration","endpoint_kind":"cpu","endpoint_condition":"evaluator-owned CPU container, offline read-only, pinned cores","gpu":"cpu","adapter_id":"images/mghafiri-qwen3.5-0.8b-decision-model/spec.json:mghafiri_adapter.JevLite","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":0.0,"calibration":73.56687971516169,"speed":71.82266186099122,"cost":79.5180630553356},"scores":{"A":0.0,"B":0.0,"C":0.0},"views":{"Balanced 33:33:33":0.0,"Emphasis on Accuracy 60:20:20":0.0,"Emphasis on Cost 20:20:60":0.0,"Emphasis on Speed 20:60:20":0.0,"Intelligence only":0.0},"composite_ci95":{"A":[0.0,0.022833461476708693],"B":[0.0,0.014952408669619277],"C":[0.0,0.01585657046993659]},"jevbench_score":0.0,"intelligence":{"I_open":3.3975872914845855,"I_sealed":-3.7143568602260615,"base":-0.158384784370738,"gap":7.111944151710647,"excess":1.9253166516314044,"penalty":1.0,"per_type_split":{"open|choice":{"cc":34.92594009023165,"tiers":{"easy":89.14681941513415,"standard":39.17338185599168,"judge":6.639004149377612,"hard":40.462201331766536},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-69.94589307967757,"tiers":{"easy":5.882352941176472,"standard":-75.75757575757575,"judge":-84.31372549019608,"hard":-75.2212389380531},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":45.212714863899684,"tiers":{"easy":28.590783410138254,"standard":48.11060070671379,"judge":43.39331585845348,"hard":49.283804060017644},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":40.43820114043159,"tiers":{"easy":50.1953125,"standard":32.08059769074034,"judge":44.34628975265018,"hard":39.24665856622116},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-86.03636363636365,"tiers":{"easy":-92.0,"standard":-80.0,"judge":-84.0,"hard":-89.0909090909091},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":34.455091915253874,"tiers":{"easy":43.77771739130435,"standard":33.51389588581023,"judge":30.51832350463354,"hard":35.54760986892834},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":73.56687971516169,"parts":{"choice":{"ece_hard":0.056884476534296013,"ece_score":88.6231046931408,"mean_tvd":0.29926050001665505,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":79.34852734573765},"noul":{"ece":0.16277070707070712,"brier":0.2519389916969697,"n":495.0,"score":67.44585858585857,"abstention_rate":0.8282828282828283},"score":{"nrps":0.16518099632828284,"top_ece":0.17834696969696973,"n":330.0,"interval90_coverage":0.8818181818181818,"interval90_width":0.6093434343434344,"score":73.90625321388889}}},"speed":{"p50_s_raw":0.579218426020816,"p95_s_raw":2.436603333125822,"p50_s_adjusted":1.3084368520416318,"p95_s_adjusted":5.023206666251644,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.00481649630541872,"basis":"documented hosted-model estimate"},"rank":79,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":79,"B":79,"C":79}},{"key":"mxbai-rerank-base-v2","display":"Mixedbread mxbai-rerank-base-v2","addendum":null,"author":"Mixedbread","repo":"https://huggingface.co/mixedbread-ai/mxbai-rerank-base-v2","class":"reranker","licence":"Apache-2.0","open":"yes","underlying":"494M cross-encoder","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (A6000), offline read-only container","gpu":"A6000","adapter_id":"images/mxbai-rerank-base-v2/spec.json:reranker_adapter.RerankerAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":0.0,"calibration":86.61210205397524,"speed":89.34435302808978,"cost":60.33589314238483},"scores":{"A":0.0,"B":0.0,"C":0.0},"views":{"Balanced 33:33:33":0.0,"Emphasis on Accuracy 60:20:20":0.0,"Emphasis on Cost 20:20:60":0.0,"Emphasis on Speed 20:60:20":0.0,"Intelligence only":0.0},"composite_ci95":{"A":[0.0,0.0],"B":[0.0,0.0],"C":[0.0,0.0]},"jevbench_score":0.0,"intelligence":{"I_open":-20.99032740064011,"I_sealed":-24.038818140610356,"base":-22.514572770625232,"gap":3.048490739970248,"excess":-2.1381367601089947,"penalty":1.0,"per_type_split":{"open|choice":{"cc":5.843867292486349,"tiers":{"easy":20.410009044317164,"standard":0.18195996880685458,"judge":4.564315352697115,"hard":5.992949471210316},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-100.0,"tiers":{"easy":-100.0,"standard":-100.0,"judge":-100.0,"hard":-100.0},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":31.185150505593313,"tiers":{"easy":6.799755544497577,"standard":31.007951785898857,"judge":37.69384193308352,"hard":32.48858003509682},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":0.6243932922656859,"tiers":{"easy":15.03906249999998,"standard":-10.029431741000643,"judge":3.268551236749129,"hard":0.36452004860269366},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-100.0,"tiers":{"easy":-100.0,"standard":-100.0,"judge":-100.0,"hard":-100.0},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":27.259152285903227,"tiers":{"easy":38.52221935703126,"standard":27.179768385394233,"judge":25.838359064936366,"hard":25.548672384100858},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":86.61210205397524,"parts":{"choice":{"ece_hard":0.028303254999639538,"ece_score":94.33934900007209,"mean_tvd":0.3377944081077518,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":80.27995409464846},"noul":{"ece":0.04816304808672505,"brier":0.2514061975269887,"n":495.0,"score":90.36739038265499,"abstention_rate":1.0},"score":{"nrps":0.18687570257877734,"top_ece":0.014672531864388688,"n":330.0,"interval90_coverage":1.0,"interval90_width":1.0,"score":89.18896168462227}}},"speed":{"p50_s_raw":0.04247652366757393,"p95_s_raw":0.1724876998458056,"p50_s_adjusted":0.23495304733514785,"p95_s_adjusted":0.4949753996916112,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.020996016009852216,"basis":"ESTIMATE: size-class proxy (deepinfra:Qwen/Qwen3-Reranker-0.6B); no exact base-model floor applies"},"rank":80,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":80,"B":80,"C":80}},{"key":"needle-3","display":"Needle 3 (Cactus, 2-bit, local CPU)","addendum":null,"author":"Cactus Compute","repo":"https://github.com/cactus-compute/needle","class":"small-tool-model","licence":"Apache-2.0 (model and package)","open":"yes","underlying":"Needle 3 (121M parameters, 2-bit)","endpoint_kind":"cpu","endpoint_condition":"evaluator-owned CPU container, offline read-only, pinned cores","gpu":"cpu","adapter_id":"images/needle-3/spec.json:jevbench.adapters.needle_local.NeedleLocalAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1101},"full_coverage":true,"support":{"choice":"label","noul":"label","score":"label"},"planned_weight_denominator":1.0,"axes":{"intelligence":0.0,"calibration":0.0,"speed":34.1252252793452,"cost":61.57165387848559},"scores":{"A":0.0,"B":0.0,"C":0.0},"views":{"Balanced 33:33:33":0.0,"Emphasis on Accuracy 60:20:20":0.0,"Emphasis on Cost 20:20:60":0.0,"Emphasis on Speed 20:60:20":0.0,"Intelligence only":0.0},"composite_ci95":{"A":[0.0,0.0],"B":[0.0,0.0],"C":[0.0,0.0]},"jevbench_score":0.0,"intelligence":{"I_open":-11.117169094537822,"I_sealed":-21.113658670994404,"base":-16.115413882766113,"gap":9.996489576456582,"excess":4.80986207637734,"penalty":1.0,"per_type_split":{"open|choice":{"cc":-3.5679770931519377,"tiers":{"easy":16.792282182695214,"standard":-7.616324408630107,"judge":-9.958506224066367,"hard":-1.8409714061888192},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-14.520372911835695,"tiers":{"easy":-17.647058823529417,"standard":-45.45454545454546,"judge":7.843137254901955,"hard":-15.04424778761062},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":-15.263157278625831,"tiers":{"easy":-2.5345622119815614,"standard":-18.816254416961108,"judge":-22.542595019659206,"hard":-11.20917917034423},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":-8.13855828680416,"tiers":{"easy":-8.39843750000002,"standard":-12.746207833371031,"judge":-3.356890459363947,"hard":-9.356014580801922},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-23.599999999999998,"tiers":{"easy":-19.999999999999996,"standard":-31.999999999999996,"judge":-24.0,"hard":-19.999999999999996},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":-31.60241772617906,"tiers":{"easy":-28.623188405797073,"standard":-27.20403022670026,"judge":-30.58129738837403,"hard":-35.312259059367726},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.33917396745932415},"noul":{"n":495,"invalid_rate":0.29292929292929293},"score":{"n":330,"invalid_rate":0.3242424242424242}},"calibration":{"score":0.0,"parts":{}},"speed":{"p50_s_raw":67.68173758895136,"p95_s_raw":142.63847913655917,"p50_s_adjusted":135.51347517790273,"p95_s_adjusted":285.4269582731183,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.019096108374384236,"basis":"ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies"},"rank":81,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":81,"B":81,"C":81}},{"key":"needle-3-tools","display":"Needle 3, options as tools (post-hoc adapter mode)","addendum":null,"author":"Cactus Compute","repo":"https://github.com/cactus-compute/needle","class":"small-tool-model","licence":"Apache-2.0 (model and package)","open":"yes","underlying":"Needle 3 (121M parameters, 2-bit)","endpoint_kind":"cpu","endpoint_condition":"evaluator-owned CPU container, offline read-only, pinned cores","gpu":"cpu","adapter_id":"images/needle-3-tools/spec.json:jevbench.adapters.needle_local.NeedleLocalAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1306},"full_coverage":true,"support":{"choice":"label","noul":"label","score":"label"},"planned_weight_denominator":1.0,"axes":{"intelligence":0.0,"calibration":0.0,"speed":40.9964345452206,"cost":61.57165387848559},"scores":{"A":0.0,"B":0.0,"C":0.0},"views":{"Balanced 33:33:33":0.0,"Emphasis on Accuracy 60:20:20":0.0,"Emphasis on Cost 20:20:60":0.0,"Emphasis on Speed 20:60:20":0.0,"Intelligence only":0.0},"composite_ci95":{"A":[0.0,0.0],"B":[0.0,0.0],"C":[0.0,0.0]},"jevbench_score":0.0,"intelligence":{"I_open":-5.786504055687074,"I_sealed":-19.897677485520102,"base":-12.842090770603589,"gap":14.111173429833027,"excess":8.924545929753783,"penalty":0.9907545407024622,"per_type_split":{"open|choice":{"cc":10.871090994383101,"tiers":{"easy":45.7340970756708,"standard":25.136469976605138,"judge":-0.6224066390041282,"hard":3.6427732079905777},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-12.378653794582998,"tiers":{"easy":-17.647058823529417,"standard":-42.42424242424242,"judge":5.882352941176472,"hard":-9.734513274336287},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":-15.851949366861326,"tiers":{"easy":11.290322580645162,"standard":-27.20848056537102,"judge":-20.576671035386607,"hard":-13.41571050308914},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":2.204169809145612,"tiers":{"easy":0.39062499999998146,"standard":-1.8791034638894857,"judge":-3.356890459363947,"hard":8.869987849331737},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-30.10909090909091,"tiers":{"easy":-43.99999999999999,"standard":-31.999999999999996,"judge":-28.000000000000004,"hard":-27.27272727272727},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":-31.788111356615012,"tiers":{"easy":-31.3405797101449,"standard":-30.982367758186413,"judge":-29.317607413647817,"hard":-34.15574402467223},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.07759699624530658},"noul":{"n":495,"invalid_rate":0.29292929292929293},"score":{"n":330,"invalid_rate":0.3363636363636363}},"calibration":{"score":0.0,"parts":{}},"speed":{"p50_s_raw":29.004772314452566,"p95_s_raw":68.26981186334741,"p50_s_adjusted":58.15954462890513,"p95_s_adjusted":136.68962372669483,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.019096108374384236,"basis":"ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies"},"rank":82,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":82,"B":82,"C":82}},{"key":"open-jev-deberta-v3-large","display":"open-jev-deberta-v3-large (local CPU)","addendum":null,"author":"Kotoba Labs","repo":"https://github.com/kotoba-lang/typed-decisions","class":"jev-rebuild","licence":"Apache-2.0 (model card); DeBERTa-v3 keeps its own terms","open":"yes","underlying":"microsoft/deberta-v3-large","endpoint_kind":"cpu","endpoint_condition":"evaluator-owned CPU container, offline read-only, pinned cores","gpu":"cpu","adapter_id":"images/open-jev-deberta-v3-large/spec.json:warm_adapter.WarmLoaded","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1610},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":0.0,"calibration":77.05564904241504,"speed":68.25469890461562,"cost":77.5922354035203},"scores":{"A":0.0,"B":0.0,"C":0.0},"views":{"Balanced 33:33:33":0.0,"Emphasis on Accuracy 60:20:20":0.0,"Emphasis on Cost 20:20:60":0.0,"Emphasis on Speed 20:60:20":0.0,"Intelligence only":0.0},"composite_ci95":{"A":[0.0,0.0],"B":[0.0,0.0],"C":[0.0,0.0]},"jevbench_score":0.0,"intelligence":{"I_open":-9.303260223347305,"I_sealed":-14.60833621498483,"base":-11.955798219166066,"gap":5.3050759916375245,"excess":0.11844849155828197,"penalty":1.0,"per_type_split":{"open|choice":{"cc":21.004312742803485,"tiers":{"easy":90.95568284594513,"standard":36.05406810501689,"judge":8.71369294605811,"hard":5.209557383470406},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-79.1568469704857,"tiers":{"easy":17.647058823529417,"standard":-78.78787878787878,"judge":-98.0392156862745,"hard":-89.38053097345133},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":30.242753557640295,"tiers":{"easy":-0.4921783980686101,"standard":33.09975231507328,"judge":31.573146619732697,"hard":35.50019237128173},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":16.66342552932574,"tiers":{"easy":26.757812499999982,"standard":13.063165044147636,"judge":24.46996466431096,"hard":10.085054678007312},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-87.38181818181818,"tiers":{"easy":-84.0,"standard":-92.0,"judge":-88.0,"hard":-85.45454545454545},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":26.893384007537954,"tiers":{"easy":40.48922199863888,"standard":27.38158587682259,"judge":23.62051138932101,"hard":25.70497803878311},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.017521902377972465},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":77.05564904241504,"parts":{"choice":{"ece_hard":0.18483169170354635,"ece_score":63.03366165929073,"mean_tvd":0.2886090069457545,"n_tvd":50.0,"n":785.0,"n_ece_hard":267.0,"score":67.08638048235764},"noul":{"ece":0.04918332987587257,"brier":0.22354630250714677,"n":495.0,"score":90.16333402482547,"abstention_rate":0.9030303030303031},"score":{"nrps":0.18992507423278554,"top_ece":0.16586513668298722,"n":330.0,"interval90_coverage":0.8424242424242424,"interval90_width":0.7098484848484842,"score":73.91723262006201}}},"speed":{"p50_s_raw":1.3912787401350215,"p95_s_raw":2.47331775734201,"p50_s_adjusted":2.932557480270043,"p95_s_adjusted":5.096635514684021,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.005583743842364532,"basis":"ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies"},"rank":83,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":83,"B":83,"C":83}},{"key":"open-jev-json-canvas-joshuasp","display":"Open Jev JSON Canvas (JoshuaSP)","addendum":null,"author":"JoshuaSP","repo":"https://github.com/JoshuaSP/open-jev","class":"jev-rebuild","licence":"MIT code; Apache-2.0 DiffusionGemma weights","open":"yes","underlying":"google/diffusiongemma-26B-A4B-it BF16, one-step JSON canvas","endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/open-jev-json-canvas-joshuasp/spec.json:joshuasp_adapter.JoshuaSPCanvasAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"label","noul":"label","score":"label"},"planned_weight_denominator":1.0,"axes":{"intelligence":77.13565407855228,"calibration":0.0,"speed":85.57288828383295,"cost":49.295219536461964},"scores":{"A":0.0,"B":0.0,"C":0.0},"views":{"Balanced 33:33:33":64.89270256009785,"Emphasis on Accuracy 60:20:20":68.58220536636948,"Emphasis on Cost 20:20:60":56.837259850231455,"Emphasis on Speed 20:60:20":71.14892802685067,"Intelligence only":74.97643178712907},"composite_ci95":{"A":[0.0,0.0],"B":[0.0,0.0],"C":[0.0,0.0]},"jevbench_score":0.0,"intelligence":{"I_open":79.15477691929314,"I_sealed":75.1165312378114,"base":77.13565407855228,"gap":4.03824568148174,"excess":-1.1483818185975023,"penalty":1.0,"per_type_split":{"open|choice":{"cc":81.04636600709232,"tiers":{"easy":98.19113656918904,"standard":92.20171562256304,"judge":89.6265560165975,"hard":64.74735605170387},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":74.9238874954648,"tiers":{"easy":82.35294117647058,"standard":84.84848484848484,"judge":84.31372549019606,"hard":61.061946902654874},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":81.49407725532232,"tiers":{"easy":93.08755760368663,"standard":90.7243816254417,"judge":80.34076015727392,"hard":74.84554280670785},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":76.36059721739608,"tiers":{"easy":94.140625,"standard":74.19062712248135,"judge":69.52296819787986,"hard":78.12879708383961},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":74.0,"tiers":{"easy":100.0,"standard":68.0,"judge":87.99999999999999,"hard":60.00000000000001},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":74.98899649603815,"tiers":{"easy":86.41304347826086,"standard":72.29219143576826,"judge":73.0412805391744,"hard":74.94217424826523},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":0.0,"parts":{}},"speed":{"p50_s_raw":0.14337781799258664,"p95_s_raw":0.2422799716354348,"p50_s_adjusted":0.4367556359851733,"p95_s_adjusted":0.6345599432708696,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.04899585591133005,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":84,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":84,"B":84,"C":84}},{"key":"openjev-verdict","display":"openJev Verdict (heman10x, ModernBERT-base 151M)","addendum":null,"author":"Hemant (heman10x)","repo":"https://github.com/Heman10x-NGU/openJev-verdict-2.0","class":"jev-rebuild","licence":"Apache-2.0","open":"yes","underlying":"GLiClass ModernBERT-base (knowledgator/gliclass-modern-base-v2.0) fine-tuned, 151M","endpoint_kind":"cpu","endpoint_condition":"evaluator-owned CPU container, offline read-only, pinned cores","gpu":"cpu","adapter_id":"images/openjev-verdict/spec.json:warm_adapter.WarmLoaded","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":0.0,"calibration":52.22236594343488,"speed":83.87645390988475,"cost":86.62313527343974},"scores":{"A":0.0,"B":0.0,"C":0.0},"views":{"Balanced 33:33:33":0.0,"Emphasis on Accuracy 60:20:20":0.0,"Emphasis on Cost 20:20:60":0.0,"Emphasis on Speed 20:60:20":0.0,"Intelligence only":0.0},"composite_ci95":{"A":[0.0,0.014760916697897527],"B":[0.0,0.0096284402945188],"C":[0.0,0.010250636595762173]},"jevbench_score":0.0,"intelligence":{"I_open":12.572554325229365,"I_sealed":-14.08853844775725,"base":-0.7579920612639421,"gap":26.661092772986613,"excess":21.47446527290737,"penalty":0.8652553472709263,"per_type_split":{"open|choice":{"cc":26.77838880692589,"tiers":{"easy":80.10250226107928,"standard":36.05406810501689,"judge":25.311203319502088,"hard":9.909909909909892},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-15.152619374378874,"tiers":{"easy":-11.764705882352944,"standard":-27.27272727272727,"judge":29.41176470588236,"hard":-43.36283185840708},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":26.091893543141083,"tiers":{"easy":-5.338195756790021,"standard":54.452603607718906,"judge":22.574273323838835,"hard":22.407276000311626},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":16.635451407617545,"tiers":{"easy":41.406249999999986,"standard":23.930269413629183,"judge":13.869257950530042,"hard":8.869987849331737},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-79.16363636363636,"tiers":{"easy":-68.0,"standard":-88.0,"judge":-88.0,"hard":-70.9090909090909},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":20.26256961274706,"tiers":{"easy":45.33950093608078,"standard":22.809338245529144,"judge":12.49636803642975,"hard":18.54460364776057},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":52.22236594343488,"parts":{"choice":{"ece_hard":0.31863114738135034,"ece_score":36.27377052372993,"mean_tvd":0.4069134621098454,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":47.791212156372694},"noul":{"ece":0.2689141261497282,"brier":0.32052442894859673,"n":495.0,"score":46.217174770054356,"abstention_rate":0.4747474747474747},"score":{"nrps":0.21716081893888597,"top_ece":0.26483248149178096,"n":330.0,"interval90_coverage":0.7848484848484848,"interval90_width":0.6196969696969695,"score":62.658710903877605}}},"speed":{"p50_s_raw":0.12610384542495012,"p95_s_raw":0.43418340694624924,"p50_s_adjusted":0.40220769084990027,"p95_s_adjusted":1.0183668138924984,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.002791871921182266,"basis":"ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies"},"rank":85,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":85,"B":85,"C":85}},{"key":"openjev-verdict-1.4","display":"openJev Verdict 1.4","addendum":null,"author":"Hemant (heman10x)","repo":"https://huggingface.co/heman10x/rlcd-modernbert-151m","class":"jev-rebuild","licence":"Apache-2.0","open":"yes","underlying":"GLiClass ModernBERT-base fine-tuned decision model, 151M; v1.4 fixed inference engine","endpoint_kind":"cpu","endpoint_condition":"evaluator-owned CPU container, offline read-only, pinned cores","gpu":"cpu","adapter_id":"images/openjev-verdict-1.4/spec.json:warm_adapter.WarmLoaded","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":0.0,"calibration":80.33031422397818,"speed":80.63780728589471,"cost":86.62313527343974},"scores":{"A":0.0,"B":0.0,"C":0.0},"views":{"Balanced 33:33:33":0.0,"Emphasis on Accuracy 60:20:20":0.0,"Emphasis on Cost 20:20:60":0.0,"Emphasis on Speed 20:60:20":0.0,"Intelligence only":0.0},"composite_ci95":{"A":[0.0,0.0],"B":[0.0,0.0],"C":[0.0,0.0]},"jevbench_score":0.0,"intelligence":{"I_open":-14.746381485338418,"I_sealed":-18.28076253742016,"base":-16.51357201137929,"gap":3.5343810520817414,"excess":-1.6522464479975012,"penalty":1.0,"per_type_split":{"open|choice":{"cc":24.287336790910803,"tiers":{"easy":80.10250226107928,"standard":37.61372498050429,"judge":17.012448132780097,"hard":9.126517822169975},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-100.0,"tiers":{"easy":-100.0,"standard":-100.0,"judge":-100.0,"hard":-100.0},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":31.47351875307393,"tiers":{"easy":0.8001436810262552,"standard":51.52721614568101,"judge":30.4854576989947,"hard":29.856059615341724},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":17.859202655617498,"tiers":{"easy":41.406249999999986,"standard":21.213493321258795,"judge":16.519434628975272,"hard":11.300121506682888},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-100.0,"tiers":{"easy":-100.0,"standard":-100.0,"judge":-100.0,"hard":-100.0},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":27.298509732122007,"tiers":{"easy":48.17842864048421,"standard":26.99406380774041,"judge":23.76842509153304,"hard":24.878316447663973},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":80.33031422397818,"parts":{"choice":{"ece_hard":0.1319904885190996,"ece_score":73.60190229618007,"mean_tvd":0.3092785057985509,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":71.33702585816249},"noul":{"ece":0.06400173539651974,"brier":0.24791482740527485,"n":495.0,"score":87.19965292069605,"abstention_rate":1.0},"score":{"nrps":0.18447119454353428,"top_ece":0.08322176379747256,"n":330.0,"interval90_coverage":0.9575757575757575,"interval90_width":0.8568181818181818,"score":82.45426389307602}}},"speed":{"p50_s_raw":0.31029939255677164,"p95_s_raw":0.48522302696947006,"p50_s_adjusted":0.7705987851135433,"p95_s_adjusted":1.12044605393894,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.002791871921182266,"basis":"ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies"},"rank":86,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":86,"B":86,"C":86}},{"key":"qwen3.8-27b","display":"Qwen3.8 27B (Chutes TEE)","addendum":null,"author":"Qwen / Chutes","repo":null,"class":"llm-baseline","licence":"open weights","open":"weights","underlying":"Qwen3.8-27B","endpoint_kind":"api","endpoint_condition":"the operator's hosted API","gpu":null,"adapter_id":"harness/specs/qwen3.8-27b.json:jevbench.adapters.openai_compat.OpenAICompatAdapter","api_flag":true,"api_exposure_note":"API measurement: the operator's endpoint received sealed item text, without answers.","status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1600},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":95.56146727664041,"calibration":98.08678902254937,"speed":56.846295929381085,"cost":0.0},"scores":{"A":0.0,"B":0.0,"C":0.0},"views":{"Balanced 33:33:33":0.0,"Emphasis on Accuracy 60:20:20":0.0,"Emphasis on Cost 20:20:60":0.0,"Emphasis on Speed 20:60:20":0.0,"Intelligence only":0.0},"composite_ci95":{"A":[0.0,0.0],"B":[0.0,0.0],"C":[0.0,0.0]},"jevbench_score":0.0,"intelligence":{"I_open":92.89676067022268,"I_sealed":98.22617388305815,"base":95.56146727664041,"gap":-5.329413212835476,"excess":-10.51604071291472,"penalty":1.0,"per_type_split":{"open|choice":{"cc":97.18121394413481,"tiers":{"easy":100.0,"standard":98.44034312451261,"judge":100.0,"hard":93.73286329808069},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":86.67881311816762,"tiers":{"easy":100.0,"standard":100.0,"judge":86.27450980392157,"hard":76.99115044247789},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":94.83025494836559,"tiers":{"easy":100.0,"standard":99.6731448763251,"judge":99.81651376146789,"hard":87.37667961165049},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":97.65836657235229,"tiers":{"easy":100.0,"standard":100.0,"judge":98.67491166077738,"hard":95.13973268529769},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":98.8,"tiers":{"easy":100.0,"standard":100.0,"judge":96.0,"hard":100.0},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":98.22015507682215,"tiers":{"easy":99.875,"standard":94.60537363560033,"judge":98.50968828980623,"hard":99.39668465690053},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.01251564455569465},"noul":{"n":495,"invalid_rate":0.02020202020202022},"score":{"n":330,"invalid_rate":0.012121212121212088}},"calibration":{"score":98.08678902254937,"parts":{"choice":{"ece_hard":0.032551433891663095,"ece_score":93.48971322166739,"mean_tvd":0.01021755439138628,"n_tvd":47.0,"n":789.0,"n_ece_hard":268.0,"score":96.23397889126437},"noul":{"ece":0.008304134010308824,"brier":0.013199678015332589,"n":485.0,"score":98.33917319793824,"abstention_rate":0.010309278350515427},"score":{"nrps":0.0008364795721932518,"top_ece":0.0027096104294481674,"n":326.0,"interval90_coverage":0.9969325153374233,"interval90_width":0.003067484662576687,"score":99.68721497844552}}},"speed":{"p50_s_raw":6.489119492471218,"p95_s_raw":31.855515367165104,"p50_s_adjusted":6.489119492471218,"p95_s_adjusted":31.855515367165104,"n":450,"adjustment":"none (hosted API, measured as is)"},"cost":{"kind":"estimate","usd_per_1000":2.1783887438423646,"basis":"ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)"},"rank":87,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":87,"B":87,"C":87}},{"key":"verdict-small","display":"verdict-small (Manavarya09, multilingual-e5-small 118M)","addendum":null,"author":"Manavarya09 (Manav)","repo":"https://github.com/Manavarya09/verdict","class":"jev-rebuild","licence":"Apache-2.0 (verdictml code and the Manav2op/verdict-small checkpoint over intfloat/multilingual-e5-small)","open":"yes","underlying":"intfloat/multilingual-e5-small (118M multilingual bi-encoder) fine-tuned on a typed-decision mix; each option is scored against the rendered state by cosine similarity, with temperature scaling and a conformal abstain set on top. One encoder pass per option, no tokens generated.","endpoint_kind":"cpu","endpoint_condition":"evaluator-owned CPU container, offline read-only, pinned cores","gpu":"cpu","adapter_id":"images/verdict-small/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":0.0,"calibration":59.45216430058828,"speed":82.05988082033076,"cost":100.0},"scores":{"A":0.0,"B":0.0,"C":0.0},"views":{"Balanced 33:33:33":0.0,"Emphasis on Accuracy 60:20:20":0.0,"Emphasis on Cost 20:20:60":0.0,"Emphasis on Speed 20:60:20":0.0,"Intelligence only":0.0},"composite_ci95":{"A":[0.0,0.005151570988376465],"B":[0.0,0.003310895006967579],"C":[0.0,0.0035774798530392115]},"jevbench_score":0.0,"intelligence":{"I_open":-0.9494028915653612,"I_sealed":-2.94840779086587,"base":-1.9489053412156156,"gap":1.999004899300509,"excess":-3.1876226007787336,"penalty":1.0,"per_type_split":{"open|choice":{"cc":26.454347123855356,"tiers":{"easy":67.44045824540248,"standard":26.696126852092533,"judge":28.423236514522838,"hard":14.61026243634937},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-55.15025318252803,"tiers":{"easy":-47.05882352941176,"standard":-45.45454545454546,"judge":-70.58823529411764,"hard":-50.442477876106196},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":25.847697383976595,"tiers":{"easy":14.758410138248824,"standard":37.14880742049469,"judge":29.26782437745743,"hard":20.404368932038853},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":12.709233032116586,"tiers":{"easy":23.828124999999982,"standard":4.912836767036478,"judge":11.219081272084816,"hard":14.94532199270962},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-43.89090909090909,"tiers":{"easy":-28.000000000000004,"standard":-28.000000000000004,"judge":-48.0,"hard":-52.727272727272734},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":22.336452686194896,"tiers":{"easy":28.33206521739132,"standard":23.274979009235974,"judge":20.83593091828141,"hard":21.493677717810357},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":59.45216430058828,"parts":{"choice":{"ece_hard":0.09814801444043317,"ece_score":80.37039711191336,"mean_tvd":0.3715248334499983,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":71.60895688345677},"noul":{"ece":0.3252131313131313,"brier":0.37707818664646464,"n":495.0,"score":34.95737373737374,"abstention_rate":0.45252525252525255},"score":{"nrps":0.2101906937752525,"top_ece":0.17700303030303038,"n":330.0,"interval90_coverage":0.8757575757575757,"interval90_width":0.7696969696969704,"score":71.79016228093434}}},"speed":{"p50_s_raw":0.03403579245787114,"p95_s_raw":1.3518648592871614,"p50_s_adjusted":0.21807158491574227,"p95_s_adjusted":2.8537297185743227,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.000866152709359606,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":88,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":88,"B":88,"C":88}},{"key":"von-395m","display":"Von (wfzyx, Option-Marker 395M)","addendum":null,"author":"wfzyx (Victor Hugo)","repo":"https://github.com/wfzyx/von","class":"jev-rebuild","licence":"Apache-2.0 code; Apache-2.0 weights (answerdotai/ModernBERT-large base)","open":"unknown","underlying":null,"endpoint_kind":"cpu","endpoint_condition":"evaluator-owned CPU container, offline read-only, pinned cores","gpu":"cpu","adapter_id":"images/von-395m/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":0.0,"calibration":83.4757036120557,"speed":75.72186940216335,"cost":82.71254751426194},"scores":{"A":0.0,"B":0.0,"C":0.0},"views":{"Balanced 33:33:33":0.0,"Emphasis on Accuracy 60:20:20":0.0,"Emphasis on Cost 20:20:60":0.0,"Emphasis on Speed 20:60:20":0.0,"Intelligence only":0.0},"composite_ci95":{"A":[0.0,0.0],"B":[0.0,0.0],"C":[0.0,0.0]},"jevbench_score":0.0,"intelligence":{"I_open":-1.8892921869454486,"I_sealed":-16.55442797958771,"base":-9.22186008326658,"gap":14.665135792642262,"excess":9.47850829256302,"penalty":0.9852149170743698,"per_type_split":{"open|choice":{"cc":34.419086270896464,"tiers":{"easy":90.95568284594513,"standard":37.61372498050429,"judge":36.721991701244825,"hard":16.960438699569117},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-78.58138911236256,"tiers":{"easy":-17.647058823529417,"standard":-60.60606060606061,"judge":-94.11764705882352,"hard":-91.1504424778761},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":38.494426280629746,"tiers":{"easy":6.3441244239631045,"standard":56.45971731448762,"judge":39.8554390563565,"hard":36.5285966460724},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":17.857166657909275,"tiers":{"easy":50.1953125,"standard":30.722209644555146,"judge":7.24381625441697,"hard":11.300121506682888},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-98.0,"tiers":{"easy":-92.0,"standard":-100.0,"judge":-96.0,"hard":-100.0},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":30.47954940332759,"tiers":{"easy":45.005797101449296,"standard":29.559109991603684,"judge":29.550168491996665,"hard":28.00524286815731},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":83.4757036120557,"parts":{"choice":{"ece_hard":0.02654584837545126,"ece_score":94.69083032490974,"mean_tvd":0.3211038334499984,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":81.29022348995495},"noul":{"ece":0.09470949494949493,"brier":0.24186929741414154,"n":495.0,"score":81.05810101010101,"abstention_rate":0.8949494949494949},"score":{"nrps":0.1641370005505052,"top_ece":0.03714363636363633,"n":330.0,"interval90_coverage":0.996969696969697,"interval90_width":0.9454545454545454,"score":88.07878633611111}}},"speed":{"p50_s_raw":0.3857649624114856,"p95_s_raw":1.3780267895897842,"p50_s_adjusted":0.9215299248229712,"p95_s_adjusted":2.9060535791795683,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.003769193349753695,"basis":"documented hosted-model estimate; no exact base-model floor applies"},"rank":89,"ranked":true,"listing":"ranked","not_ranked_because":null,"ranks":{"A":89,"B":89,"C":89}},{"key":"classifier-dev-fast","display":"classifier.dev (fast tier)","addendum":null,"author":"mrmps (@michael_chomsky)","repo":"https://classifier.dev","class":"jev-service","licence":"MIT (code); hosted service","open":"no","underlying":"Jev (TypeSafe) behind classifier.dev's zero-shot classification API","endpoint_kind":"api","endpoint_condition":"the operator's hosted API","gpu":null,"adapter_id":"harness/specs/classifier-dev-fast.json:jevbench.adapters.classifier_dev.ClassifierDevAdapter","api_flag":true,"api_exposure_note":"API measurement: the operator's endpoint received sealed item text, without answers.","status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":75.80855995014088,"calibration":89.18872642455774,"speed":81.99020200942311,"cost":58.89475669158328},"scores":{"A":74.65566746650249,"B":74.88343210437176,"C":74.65566746650249},"views":{"Balanced 33:33:33":70.80959108698245,"Emphasis on Accuracy 60:20:20":72.7279212277414,"Emphasis on Cost 20:20:60":65.50845755891696,"Emphasis on Speed 20:60:20":74.89480959179811,"Intelligence only":75.80855995014088},"composite_ci95":{"A":[73.50372056154933,75.20247331380175],"B":[73.39160786384885,75.83326867034351],"C":[73.50372056154933,75.20247331380175]},"jevbench_score":74.65566746650249,"intelligence":{"I_open":74.48059742656369,"I_sealed":77.13652247371806,"base":75.80855995014088,"gap":-2.655925047154369,"excess":-7.842552547233612,"penalty":1.0,"per_type_split":{"open|choice":{"cc":84.16704391568094,"tiers":{"easy":98.19113656918904,"standard":96.88068624902522,"judge":93.7759336099585,"hard":67.09753231492361},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":56.779455144889816,"tiers":{"easy":76.47058823529412,"standard":66.66666666666667,"judge":80.3921568627451,"hard":29.20353982300885},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":82.49529321912031,"tiers":{"easy":77.05069124423962,"standard":87.68109540636043,"judge":88.3224115334207,"hard":76.89320388349515},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":85.10653662378164,"tiers":{"easy":97.0703125,"standard":86.41611953814805,"judge":81.4487632508834,"hard":84.2041312272175},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":64.43636363636364,"tiers":{"easy":84.00000000000001,"standard":80.0,"judge":68.0,"hard":49.09090909090909},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":81.86666716100889,"tiers":{"easy":86.22282608695653,"standard":77.18316667938836,"judge":84.51979780960403,"hard":81.12952968388589},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":89.18872642455774,"parts":{"choice":{"ece_hard":0.051448419210152085,"ece_score":89.71031615796959,"mean_tvd":0.3067600453378805,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":79.51715581209078},"noul":{"ece":0.03414141414141409,"brier":0.07133171717171723,"n":495.0,"score":93.17171717171718,"abstention_rate":0.15353535353535352},"score":{"nrps":0.04169966544848548,"top_ece":0.03037710437710453,"n":330.0,"interval90_coverage":0.9818181818181818,"interval90_width":0.21111111111111108,"score":94.87730628986527}}},"speed":{"p50_s_raw":0.5177607946097851,"p95_s_raw":1.221379529684782,"p50_s_adjusted":0.5177607946097851,"p95_s_adjusted":1.221379529684782,"n":450,"adjustment":"none (hosted API, measured as is)"},"cost":{"kind":"estimate","usd_per_1000":0.02345172413793104,"basis":"ESTIMATE (proxy tokens, I-2): operator list price: higher of 19 Sep plan cost and 26 Sep usage tariff USD 0.042/M input (rule 1.2); no exact base-model floor applies"},"rank":null,"ranked":false,"listing":"honorable_mention","not_ranked_because":"runs on Jev (TypeSafe) - listed, not ranked (honorable mention, as in v1.4.2)","ranks":{"A":null,"B":null,"C":null}},{"key":"jevk5-v0.3-4b","display":"JevK5 v0.3 (4B)","addendum":{"id":"A1","release":"v1.5","label":"v1.5 roster addendum A1"},"author":"unknown","repo":null,"class":"unclassified","licence":"not recorded","open":"unknown","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/jevk5-v0.3-4b/spec.json:jevk5_v15.JevK5Warm","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":56.27051532364801,"calibration":88.34394935502905,"speed":93.55214886599305,"cost":63.07359395281681},"scores":{"A":71.89794783714967,"B":68.11459443579027,"C":63.237658174056016},"views":{"Balanced 33:33:33":67.69714212183655,"Emphasis on Accuracy 60:20:20":62.611440918908144,"Emphasis on Cost 20:20:60":65.76869806491888,"Emphasis on Speed 20:60:20":76.11106829175051,"Intelligence only":56.270515323648006},"composite_ci95":{"A":[69.39190737629329,72.94757443195746],"B":[64.93425490805247,69.81993373931475],"C":[51.342990288339934,72.12390553992212]},"jevbench_score":71.89794783714967,"intelligence":{"I_open":61.599649589732365,"I_sealed":50.94138105756366,"base":56.27051532364801,"gap":10.658268532168705,"excess":5.471641032089463,"penalty":1.0,"per_type_split":{"open|choice":{"cc":80.25232318940107,"tiers":{"easy":96.38227313837805,"standard":79.7244606186639,"judge":88.58921161825727,"hard":70.23110066588326},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":42.32817503509851,"tiers":{"easy":64.70588235294117,"standard":30.303030303030297,"judge":62.745098039215684,"hard":27.43362831858407},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":62.21845054469753,"tiers":{"easy":27.990399176120206,"standard":78.608761269458,"judge":61.898387855725666,"hard":62.82035504119053},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":74.2487837985823,"tiers":{"easy":76.56249999999999,"standard":74.19062712248135,"judge":74.82332155477033,"hard":73.2685297691373},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":14.981818181818179,"tiers":{"easy":60.00000000000001,"standard":-7.9999999999999964,"judge":28.000000000000004,"hard":5.454545454545445},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":63.593541192290516,"tiers":{"easy":78.74669580536042,"standard":69.23663981627274,"judge":60.357844150093996,"hard":59.410476008679325},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":88.34394935502905,"parts":{"choice":{"ece_hard":0.03780043689998049,"ece_score":92.43991262000391,"mean_tvd":0.2639746135435604,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":83.02122563282393},"noul":{"ece":0.030562476858007503,"brier":0.09874192080011364,"n":495.0,"score":93.8875046283985,"abstention_rate":0.2909090909090909},"score":{"nrps":0.09431639573309676,"top_ece":0.0716106240948041,"n":330.0,"interval90_coverage":0.9363636363636364,"interval90_width":0.38863636363636356,"score":88.12311780386474}}},"speed":{"p50_s_raw":0.01584504608763382,"p95_s_raw":0.04645736991078593,"p50_s_adjusted":0.18169009217526763,"p95_s_adjusted":0.24291473982157186,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.01701689039408867,"basis":"documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)"},"rank":null,"ranked":false,"listing":"addendum","not_ranked_because":"v1.5 roster addendum A1: complete run, scored with the frozen v1.5.0 median gap. Headline A score 71.90 would place about #4; secondary B score would place about #5. Roster addendum rows stay outside the v1.5.0 order and tie markers.","would_place_A":4,"would_place_B":5,"ranks":{"A":null,"B":null,"C":null}},{"key":"plumb-4b","display":"Plumb-4B (crh225, JevK5 v0.2 + LoRA)","addendum":{"id":"A1","release":"v1.5","label":"v1.5 roster addendum A1"},"author":"unknown","repo":null,"class":"unclassified","licence":"not recorded","open":"unknown","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/plumb-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":55.84843638193568,"calibration":87.44416146719458,"speed":93.45235823893108,"cost":63.07359395281681},"scores":{"A":71.56069240662383,"B":67.74864628766687,"C":62.00033986975725},"views":{"Balanced 33:33:33":67.47526154024382,"Emphasis on Accuracy 60:20:20":62.288260696856106,"Emphasis on Cost 20:20:60":65.64287436007193,"Emphasis on Speed 20:60:20":75.91628888044532,"Intelligence only":55.84843638193568},"composite_ci95":{"A":[69.19832449519649,72.72798975349464],"B":[64.67900356778624,69.55675259096279],"C":[50.654423777900476,70.8909332025414]},"jevbench_score":71.56069240662383,"intelligence":{"I_open":60.882134008279415,"I_sealed":50.81473875559194,"base":55.84843638193568,"gap":10.067395252687476,"excess":4.880767752608233,"penalty":1.0,"per_type_split":{"open|choice":{"cc":80.42963055688979,"tiers":{"easy":98.19113656918904,"standard":81.28411749415127,"judge":89.6265560165975,"hard":68.66431649040345},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":42.95206095310208,"tiers":{"easy":64.70588235294117,"standard":36.36363636363635,"judge":60.7843137254902,"hard":27.43362831858407},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":59.26471051484638,"tiers":{"easy":28.14668835825308,"standard":75.0123339949518,"judge":58.475823481817436,"hard":59.762069588713686},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":73.29707817497771,"tiers":{"easy":85.3515625,"standard":70.11546298392577,"judge":68.19787985865725,"hard":75.69866342648847},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":16.58181818181818,"tiers":{"easy":68.0,"standard":8.000000000000007,"judge":19.999999999999996,"hard":5.454545454545445},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":62.565319909979934,"tiers":{"easy":73.45705646633482,"standard":68.81432190455739,"judge":63.67451290834598,"hard":55.88599002482795},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":87.44416146719458,"parts":{"choice":{"ece_hard":0.06834988430518965,"ece_score":86.33002313896208,"mean_tvd":0.17367101892281844,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":84.48146062334013},"noul":{"ece":0.04175922105446306,"brier":0.09947893375488488,"n":495.0,"score":91.64815578910739,"abstention_rate":0.2828282828282829},"score":{"nrps":0.09880470839715763,"top_ece":0.08856896591005901,"n":330.0,"interval90_coverage":0.906060606060606,"interval90_width":0.4032828282828283,"score":86.20286798913622}}},"speed":{"p50_s_raw":0.01726541091920808,"p95_s_raw":0.0473672681429889,"p50_s_adjusted":0.18453082183841615,"p95_s_adjusted":0.2447345362859778,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.01701689039408867,"basis":"documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)"},"rank":null,"ranked":false,"listing":"addendum","not_ranked_because":"v1.5 roster addendum A1: complete run, scored with the frozen v1.5.0 median gap. Headline A score 71.56 would place about #4; secondary B score would place about #5. Roster addendum rows stay outside the v1.5.0 order and tie markers.","would_place_A":4,"would_place_B":5,"ranks":{"A":null,"B":null,"C":null}},{"key":"decision-4b-v12","display":"Decision 4B v1.2 (FlyMyJev, Qwen3.5-4B + LoRA)","addendum":{"id":"A1","release":"v1.5","label":"v1.5 roster addendum A1"},"author":"unknown","repo":null,"class":"unclassified","licence":"not recorded","open":"unknown","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/decision-4b-v12/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":53.663766457491015,"calibration":88.56416661568062,"speed":93.50125558915451,"cost":63.07359395281681},"scores":{"A":70.82738025747577,"B":66.56913664963011,"C":56.657966014846686},"views":{"Balanced 33:33:33":66.39505737102407,"Emphasis on Accuracy 60:20:20":60.64047420136369,"Emphasis on Cost 20:20:60":65.02535990714459,"Emphasis on Speed 20:60:20":75.10419953993227,"Intelligence only":53.66376645749102},"composite_ci95":{"A":[68.51591643379686,71.96768836941042],"B":[63.67943198254703,68.39175169030159],"C":[47.580497523470044,65.08783016151382]},"jevbench_score":70.82738025747577,"intelligence":{"I_open":56.26310318837419,"I_sealed":51.06442972660784,"base":53.663766457491015,"gap":5.198673461766347,"excess":0.0120459616871047,"penalty":1.0,"per_type_split":{"open|choice":{"cc":80.86757187722047,"tiers":{"easy":96.38227313837805,"standard":82.84377436963868,"judge":92.73858921161826,"hard":67.09753231492361},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":20.127616613821715,"tiers":{"easy":58.823529411764696,"standard":39.393939393939405,"judge":5.882352941176472,"hard":11.504424778761058},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":67.79412107408038,"tiers":{"easy":47.702439113741114,"standard":79.25132012928306,"judge":71.68112016355602,"hard":64.1731927194571},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":73.75526883039994,"tiers":{"easy":73.63281249999999,"standard":78.2657912610369,"judge":68.19787985865725,"hard":75.69866342648847},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":16.436363636363634,"tiers":{"easy":60.00000000000001,"standard":4.0000000000000036,"judge":19.999999999999996,"hard":9.090909090909083},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":63.00165671305996,"tiers":{"easy":78.49758935921852,"standard":68.57556229972187,"judge":66.47629048096655,"hard":53.73474543225941},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":88.56416661568062,"parts":{"choice":{"ece_hard":0.04734383561120563,"ece_score":90.53123287775887,"mean_tvd":0.21448997597778782,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":84.54111763999003},"noul":{"ece":0.05692384040660478,"brier":0.0977795596104323,"n":495.0,"score":88.61523191867904,"abstention_rate":0.36969696969696975},"score":{"nrps":0.07819475027704766,"top_ece":0.035541121977748294,"n":330.0,"interval90_coverage":0.9696969696969697,"interval90_width":0.4462121212121212,"score":92.5361502883728}}},"speed":{"p50_s_raw":0.016381043009459972,"p95_s_raw":0.04716824610950424,"p50_s_adjusted":0.18276208601891994,"p95_s_adjusted":0.2443364922190085,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.01701689039408867,"basis":"documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)"},"rank":null,"ranked":false,"listing":"addendum","not_ranked_because":"v1.5 roster addendum A1: complete run, scored with the frozen v1.5.0 median gap. Headline A score 70.83 would place about #6; secondary B score would place about #7. Roster addendum rows stay outside the v1.5.0 order and tie markers.","would_place_A":6,"would_place_B":7,"ranks":{"A":null,"B":null,"C":null}},{"key":"imajev-4b-rtx5090-a2","display":"Imajev-4B (RTX 5090)","addendum":{"id":"A2","release":"v1.5","label":"v1.5 roster addendum A2"},"author":"unknown","repo":null,"class":"unclassified","licence":"not recorded","open":"unknown","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX 5090), offline read-only container","gpu":"RTX 5090","adapter_id":"A2 reviewed bundle","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":53.47383727839022,"calibration":88.11694149062471,"speed":91.13208929836159,"cost":63.26458220826236},"scores":{"A":70.38659212275981,"B":66.19909019518123,"C":55.907508463046334},"views":{"Balanced 33:33:33":65.96240864336546,"Emphasis on Accuracy 60:20:20":60.32678975105616,"Emphasis on Cost 20:20:60":64.85613036338354,"Emphasis on Speed 20:60:20":74.15469284216614,"Intelligence only":53.47383727839022},"composite_ci95":{"A":[67.79966861439989,71.61191717463264],"B":[63.11559140704828,68.08825963056043],"C":[47.08310320444435,64.45534727420464]},"jevbench_score":70.38659212275981,"intelligence":{"I_open":56.24619253993687,"I_sealed":50.70148201684356,"base":53.47383727839022,"gap":5.544710523093308,"excess":0.358083023014065,"penalty":1.0,"per_type_split":{"open|choice":{"cc":80.11776048417694,"tiers":{"easy":98.19113656918904,"standard":89.08240187158823,"judge":85.47717842323651,"hard":67.09753231492361},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":21.823071948006877,"tiers":{"easy":64.70588235294117,"standard":51.515151515151516,"judge":27.450980392156854,"hard":-7.964601769911505},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":66.79774518762679,"tiers":{"easy":36.12484114152319,"standard":79.14114403997071,"judge":68.57269580952628,"hard":66.96305880655612},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":82.32588710912236,"tiers":{"easy":88.28125,"standard":76.9074032148517,"judge":81.4487632508834,"hard":84.2041312272175},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":7.563636363636362,"tiers":{"easy":43.99999999999999,"standard":4.0000000000000036,"judge":19.999999999999996,"hard":-9.090909090909093},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":62.21492257777197,"tiers":{"easy":73.37790947410274,"standard":65.41303393458439,"judge":62.0389806565921,"hard":57.95707661616796},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":88.11694149062471,"parts":{"choice":{"ece_hard":0.06801188748719926,"ece_score":86.39762250256014,"mean_tvd":0.2029350870949496,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":83.05205689653259},"noul":{"ece":0.04795614076750405,"brier":0.11127612689924851,"n":495.0,"score":90.40877184649919,"abstention_rate":0.3737373737373737},"score":{"nrps":0.08668615613253966,"top_ece":0.04775696464530638,"n":330.0,"interval90_coverage":0.9393939393939394,"interval90_width":0.3345959595959594,"score":90.88999572884238}}},"speed":{"p50_s_raw":0.041955091524869204,"p95_s_raw":0.08970695780590177,"p50_s_adjusted":0.2339101830497384,"p95_s_adjusted":0.32941391561180355,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.016769261083743846,"basis":"ESTIMATE (I-2): measured input tokens; zero generated output tokens for signed logits readout. M2 floor uses the 25 Sep 2026 DeepInfra Qwen3.5-4B snapshot rates (USD 0.03/M input, USD 0.15/M output); the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen3.5-9B."},"rank":null,"ranked":false,"listing":"addendum","not_ranked_because":"v1.5 roster addendum A2: complete run, scored with the frozen v1.5.0 median gap. Headline A score 70.39 would place about #6; secondary B score would place about #7. Roster addendum rows stay outside the v1.5.0 order and tie markers.","would_place_A":6,"would_place_B":7,"ranks":{"A":null,"B":null,"C":null}},{"key":"decision-4b-v11","display":"Decision 4B v1.1 (FlyMyJev, Qwen3.5-4B + LoRA)","addendum":{"id":"A1","release":"v1.5","label":"v1.5 roster addendum A1"},"author":"unknown","repo":null,"class":"unclassified","licence":"not recorded","open":"unknown","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/decision-4b-v11/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":53.10782790674597,"calibration":87.31445358951783,"speed":93.52161857471674,"cost":63.07359395281681},"scores":{"A":70.38575169491591,"B":66.08572948524592,"C":55.144135275761755},"views":{"Balanced 33:33:33":66.11304184835826,"Emphasis on Accuracy 60:20:20":60.21480463822364,"Emphasis on Cost 20:20:60":64.8627752105624,"Emphasis on Speed 20:60:20":74.89261623851299,"Intelligence only":53.107827906745975},"composite_ci95":{"A":[66.85836636993423,71.56623969374822],"B":[62.10775126922529,67.92588259628072],"C":[46.42942109023209,63.50687834004045]},"jevbench_score":70.38575169491591,"intelligence":{"I_open":54.6540553015887,"I_sealed":51.561600511903244,"base":53.10782790674597,"gap":3.092454789685455,"excess":-2.0941727103937877,"penalty":1.0,"per_type_split":{"open|choice":{"cc":80.1205807529816,"tiers":{"easy":98.19113656918904,"standard":81.28411749415127,"judge":87.55186721991701,"hard":69.44770857814335},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":18.021548120454945,"tiers":{"easy":52.941176470588225,"standard":42.42424242424243,"judge":5.882352941176472,"hard":6.194690265486735},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":65.82003703132958,"tiers":{"easy":46.64708904955269,"standard":78.7058833142304,"judge":67.933060317906,"hard":62.585583420391046},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":74.76525759162931,"tiers":{"easy":88.28125,"standard":75.5490151686665,"judge":66.87279151943464,"hard":76.91373025516404},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":15.745454545454546,"tiers":{"easy":60.00000000000001,"standard":4.0000000000000036,"judge":8.000000000000007,"hard":16.36363636363636},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":64.17408939862588,"tiers":{"easy":82.70764582802245,"standard":70.37001442323314,"judge":64.58457567971891,"hard":56.134873068153325},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":87.31445358951783,"parts":{"choice":{"ece_hard":0.061003907977028454,"ece_score":87.7992184045943,"mean_tvd":0.2162431550144764,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":83.08745145157333},"noul":{"ece":0.05942903872334748,"brier":0.09970492807158093,"n":495.0,"score":88.11419225533051,"abstention_rate":0.38181818181818183},"score":{"nrps":0.08303471757831037,"top_ece":0.05106547059434831,"n":330.0,"interval90_coverage":0.9666666666666667,"interval90_width":0.4497474747474746,"score":90.74171706164965}}},"speed":{"p50_s_raw":0.016276915499474853,"p95_s_raw":0.04673548646387644,"p50_s_adjusted":0.1825538309989497,"p95_s_adjusted":0.24347097292775288,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.01701689039408867,"basis":"documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)"},"rank":null,"ranked":false,"listing":"addendum","not_ranked_because":"v1.5 roster addendum A1: complete run, scored with the frozen v1.5.0 median gap. Headline A score 70.39 would place about #6; secondary B score would place about #7. Roster addendum rows stay outside the v1.5.0 order and tie markers.","would_place_A":6,"would_place_B":7,"ranks":{"A":null,"B":null,"C":null}},{"key":"surogate-rune-26b-a4b-v3-rtxpro6000-a2","display":"Surogate Rune 26B-A4B v3 (RTX PRO 6000)","addendum":{"id":"A2","release":"v1.5","label":"v1.5 roster addendum A2"},"author":"unknown","repo":null,"class":"unclassified","licence":"not recorded","open":"unknown","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX PRO 6000), offline read-only container","gpu":"RTX PRO 6000","adapter_id":"A2 reviewed bundle","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":69.7384372659144,"calibration":88.29679782691991,"speed":85.9674611821838,"cost":48.9678579741414},"scores":{"A":66.47011066179378,"B":66.55345952957438,"C":66.47011066179378},"views":{"Balanced 33:33:33":62.02253148171242,"Emphasis on Accuracy 60:20:20":63.881582091786825,"Emphasis on Cost 20:20:60":54.973713258577675,"Emphasis on Speed 20:60:20":68.84661497656214,"Intelligence only":66.88895581073571},"composite_ci95":{"A":[65.43169342866265,67.00710945239213],"B":[65.10706397319322,67.52972503849682],"C":[65.43169342866265,67.00710945239213]},"jevbench_score":66.47011066179378,"intelligence":{"I_open":68.95080025326385,"I_sealed":70.52607427856495,"base":69.7384372659144,"gap":-1.5752740253010984,"excess":-6.761901525380341,"penalty":1.0,"per_type_split":{"open|choice":{"cc":81.98573910797714,"tiers":{"easy":98.19113656918904,"standard":95.32102937353783,"judge":88.58921161825727,"hard":66.31414022718371},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":45.58752543656239,"tiers":{"easy":76.47058823529412,"standard":45.45454545454546,"judge":66.66666666666667,"hard":22.123893805309724},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":79.27913621525204,"tiers":{"easy":77.45483328730258,"standard":87.3570213774979,"judge":80.61432644653624,"hard":74.69487669265334},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":82.17410009259264,"tiers":{"easy":94.140625,"standard":79.6241793072221,"judge":78.79858657243817,"hard":82.98906439854193},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":49.52727272727273,"tiers":{"easy":84.00000000000001,"standard":43.99999999999999,"judge":52.0,"hard":41.81818181818182},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":79.87685001582949,"tiers":{"easy":81.44270010435835,"standard":80.09762263276632,"judge":83.36030214546362,"hard":76.76241208800323},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":88.29679782691991,"parts":{"choice":{"ece_hard":0.048130505772540376,"ece_score":90.37389884549192,"mean_tvd":0.26981249787409634,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":81.69632452904114},"noul":{"ece":0.041444379611227856,"brier":0.06554161668204889,"n":495.0,"score":91.71112407775442,"abstention_rate":0.24444444444444446},"score":{"nrps":0.04234138893172694,"top_ece":0.06399985679449458,"n":330.0,"interval90_coverage":0.9939393939393939,"interval90_width":0.28257575757575754,"score":91.4829448739642}}},"speed":{"p50_s_raw":0.10046782437711954,"p95_s_raw":0.28557565337978286,"p50_s_adjusted":0.3509356487542391,"p95_s_adjusted":0.7211513067595657,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.050242518472906394,"basis":"ESTIMATE: M2 base-model market reference from the frozen 25 Sep 2026 OpenRouter snapshot applied to measured tokens."},"rank":null,"ranked":false,"listing":"addendum","not_ranked_because":"v1.5 roster addendum A2: complete run, scored with the frozen v1.5.0 median gap. Headline A score 66.47 would place about #11; secondary B score would place about #7. Roster addendum rows stay outside the v1.5.0 order and tie markers.","would_place_A":11,"would_place_B":7,"ranks":{"A":null,"B":null,"C":null}},{"key":"autojev-27b","display":"AutoJev-27B (denis-pplx, Qwen3.8-27B)","addendum":{"id":"A1","release":"v1.5","label":"v1.5 roster addendum A1"},"author":"unknown","repo":null,"class":"unclassified","licence":"not recorded","open":"unknown","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/autojev-27b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":72.78028107753738,"calibration":87.70317647540496,"speed":87.62166952486342,"cost":29.365937020749882},"scores":{"A":19.54135920374164,"B":20.447669591289714,"C":19.54135920374164},"views":{"Balanced 33:33:33":17.478538208811404,"Emphasis on Accuracy 60:20:20":19.89620159601279,"Emphasis on Cost 20:20:60":13.547178051841165,"Emphasis on Speed 20:60:20":21.025141031925013,"Intelligence only":25.10507053670285},"composite_ci95":{"A":[19.299804948470786,19.631386859392],"B":[20.095989263006373,20.648563859347476],"C":[19.299804948470786,19.631386859392]},"jevbench_score":19.54135920374164,"intelligence":{"I_open":71.91293867189668,"I_sealed":73.64762348317807,"base":72.78028107753738,"gap":-1.734684811281383,"excess":-6.921312311360626,"penalty":1.0,"per_type_split":{"open|choice":{"cc":85.2388568573883,"tiers":{"easy":96.38227313837805,"standard":95.32102937353783,"judge":94.81327800829875,"hard":70.23110066588326},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":50.436010285047246,"tiers":{"easy":76.47058823529412,"standard":69.6969696969697,"judge":66.66666666666667,"hard":22.123893805309724},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":80.06394887325453,"tiers":{"easy":80.58788363302166,"standard":86.65872826406437,"judge":80.03010394222157,"hard":76.66095918618254},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":82.24130918153313,"tiers":{"easy":91.2109375,"standard":80.98256735340728,"judge":77.47349823321555,"hard":84.2041312272175},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":62.0,"tiers":{"easy":84.00000000000001,"standard":52.0,"judge":63.99999999999999,"hard":60.00000000000001},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":76.70156126800109,"tiers":{"easy":81.85626924529423,"standard":75.87669579367935,"judge":76.44648330538433,"hard":76.01662548280125},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":87.70317647540496,"parts":{"choice":{"ece_hard":0.04212943593705838,"ece_score":91.57411281258833,"mean_tvd":0.25162423089203245,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":83.20584486169254},"noul":{"ece":0.04052854529071123,"brier":0.05503997046028658,"n":495.0,"score":91.89429094185775,"abstention_rate":0.21010101010101012},"score":{"nrps":0.04502967345008432,"top_ece":0.09739122704831175,"n":330.0,"interval90_coverage":0.9878787878787879,"interval90_width":0.29267676767676765,"score":88.00939362266462}}},"speed":{"p50_s_raw":0.10088440257823095,"p95_s_raw":0.17077954066335213,"p50_s_adjusted":0.3517688051564619,"p95_s_adjusted":0.49155908132670423,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.22618758620689655,"basis":"documented hosted-model estimate"},"rank":null,"ranked":false,"listing":"addendum","not_ranked_because":"v1.5 roster addendum A1: complete run, scored with the frozen v1.5.0 median gap. Headline A score 19.54 would place about #43; secondary B score would place about #40. Roster addendum rows stay outside the v1.5.0 order and tie markers.","would_place_A":43,"would_place_B":40,"ranks":{"A":null,"B":null,"C":null}},{"key":"autojev-27b-rtxpro6000-a2","display":"AutoJev-27B (RTX PRO 6000)","addendum":{"id":"A2","release":"v1.5","label":"v1.5 roster addendum A2"},"author":"unknown","repo":null,"class":"unclassified","licence":"not recorded","open":"unknown","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (RTX PRO 6000), offline read-only container","gpu":"RTX PRO 6000","adapter_id":"A2 reviewed bundle","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":72.7504220700132,"calibration":86.68041666712992,"speed":87.06975289886662,"cost":29.365937020749882},"scores":{"A":19.482719520484327,"B":20.394920235576496,"C":19.482719520484327},"views":{"Balanced 33:33:33":17.455546885699814,"Emphasis on Accuracy 60:20:20":19.875735473140935,"Emphasis on Cost 20:20:60":13.538885114405412,"Emphasis on Speed 20:60:20":20.968224367337445,"Intelligence only":25.094770872027855},"composite_ci95":{"A":[19.262439246036728,19.59873421101785],"B":[20.06322075545375,20.611081472646248],"C":[19.262439246036728,19.59873421101785]},"jevbench_score":19.482719520484327,"intelligence":{"I_open":72.0148638915178,"I_sealed":73.4859802485086,"base":72.7504220700132,"gap":-1.471116356990791,"excess":-6.657743857070034,"penalty":1.0,"per_type_split":{"open|choice":{"cc":84.92550002229234,"tiers":{"easy":96.38227313837805,"standard":95.32102937353783,"judge":94.81327800829875,"hard":69.44770857814335},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":51.14397488681716,"tiers":{"easy":76.47058823529412,"standard":69.6969696969697,"judge":66.66666666666667,"hard":23.893805309734507},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":79.97511676544391,"tiers":{"easy":80.15537374983703,"standard":86.72749010669713,"judge":79.87342491381291,"hard":76.63013473744229},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":81.69795396305905,"tiers":{"easy":91.2109375,"standard":78.2657912610369,"judge":77.47349823321555,"hard":84.2041312272175},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":62.0,"tiers":{"easy":84.00000000000001,"standard":52.0,"judge":63.99999999999999,"hard":60.00000000000001},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":76.75998678246677,"tiers":{"easy":81.77890543489788,"standard":76.16408820390532,"judge":76.5872741824193,"hard":75.9327408586753},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":86.68041666712992,"parts":{"choice":{"ece_hard":0.05563197319122527,"ece_score":88.87360536175495,"mean_tvd":0.25077706263870864,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":81.89794954894205},"noul":{"ece":0.04353587849034292,"brier":0.05491962545051717,"n":495.0,"score":91.29282430193142,"abstention_rate":0.2080808080808081},"score":{"nrps":0.04503821273971415,"top_ece":0.10897613212498017,"n":330.0,"interval90_coverage":0.9878787878787879,"interval90_width":0.29318181818181815,"score":86.85047615051627}}},"speed":{"p50_s_raw":0.08811413822695613,"p95_s_raw":0.22593528539873534,"p50_s_adjusted":0.3262282764539123,"p95_s_adjusted":0.6018705707974706,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.22618758620689655,"basis":"ESTIMATE: M2 base-model market reference from the frozen 25 Sep 2026 OpenRouter snapshot applied to measured tokens."},"rank":null,"ranked":false,"listing":"addendum","not_ranked_because":"v1.5 roster addendum A2: complete run, scored with the frozen v1.5.0 median gap. Headline A score 19.48 would place about #43; secondary B score would place about #40. Roster addendum rows stay outside the v1.5.0 order and tie markers.","would_place_A":43,"would_place_B":40,"ranks":{"A":null,"B":null,"C":null}},{"key":"eikos-27b","display":"Eikos-27B (caiovicentino1, Qwen3.8-27B)","addendum":{"id":"A1","release":"v1.5","label":"v1.5 roster addendum A1"},"author":"unknown","repo":null,"class":"unclassified","licence":"not recorded","open":"unknown","underlying":null,"endpoint_kind":"gpu","endpoint_condition":"evaluator-owned Lium GPU pod (H100), offline read-only container","gpu":"H100","adapter_id":"images/eikos-27b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":false,"api_exposure_note":null,"status":{"status":"complete","rows":1624,"missing":0,"answered_ok":1624},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":75.08487940331634,"calibration":86.28582452360858,"speed":87.62506251841538,"cost":28.689752882469676},"scores":{"A":18.503023570753697,"B":19.483126908668613,"C":18.503023570753697},"views":{"Balanced 33:33:33":16.576368026746824,"Emphasis on Accuracy 60:20:20":19.092466134747955,"Emphasis on Cost 20:20:60":12.731910464092147,"Emphasis on Speed 20:60:20":19.97561012269975,"Intelligence only":24.721003373666086},"composite_ci95":{"A":[18.282206247053008,18.598674719666235],"B":[19.15860082154746,19.66517783572774],"C":[18.282206247053008,18.598674719666235]},"jevbench_score":18.503023570753697,"intelligence":{"I_open":74.1460564615501,"I_sealed":76.02370234508257,"base":75.08487940331634,"gap":-1.877645883532466,"excess":-7.064273383611709,"penalty":1.0,"per_type_split":{"open|choice":{"cc":87.62039899732592,"tiers":{"easy":98.19113656918904,"standard":92.20171562256304,"judge":92.73858921161826,"hard":78.84841363102232},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":56.46932626630701,"tiers":{"easy":76.47058823529412,"standard":69.6969696969697,"judge":49.01960784313726,"hard":50.44247787610618},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":78.34844412101737,"tiers":{"easy":77.79806316637085,"standard":83.74739299176998,"judge":80.10505471948588,"hard":74.46910697545131},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":88.26916117896289,"tiers":{"easy":91.2109375,"standard":85.05773149196287,"judge":86.74911660777386,"hard":90.27946537059537},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":64.14545454545456,"tiers":{"easy":92.0,"standard":60.00000000000001,"judge":68.0,"hard":56.363636363636374},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":75.65649131083029,"tiers":{"easy":79.54885680667138,"standard":74.95659667272105,"judge":76.56023976602533,"hard":74.35553591452833},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.0},"noul":{"n":495,"invalid_rate":0.0},"score":{"n":330,"invalid_rate":0.0}},"calibration":{"score":86.28582452360858,"parts":{"choice":{"ece_hard":0.03693915188097349,"ece_score":92.6121696238053,"mean_tvd":0.2546374930060419,"n_tvd":50.0,"n":799.0,"n_ece_hard":277.0,"score":83.57421016160055},"noul":{"ece":0.07420246840426414,"brier":0.054408630663178775,"n":495.0,"score":85.15950631914717,"abstention_rate":0.1838383838383838},"score":{"nrps":0.04704954457108033,"top_ece":0.0752376568136793,"n":330.0,"interval90_coverage":0.9636363636363636,"interval90_width":0.24343434343434348,"score":90.12375709007804}}},"speed":{"p50_s_raw":0.10209655796643347,"p95_s_raw":0.16890664728125523,"p50_s_adjusted":0.35419311593286695,"p95_s_adjusted":0.48781329456251044,"n":450,"adjustment":"x2 + 0.15 s (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.23823646551724137,"basis":"documented hosted-model estimate"},"rank":null,"ranked":false,"listing":"addendum","not_ranked_because":"v1.5 roster addendum A1: complete run, scored with the frozen v1.5.0 median gap. Headline A score 18.50 would place about #44; secondary B score would place about #40. Roster addendum rows stay outside the v1.5.0 order and tie markers.","would_place_A":44,"would_place_B":40,"ranks":{"A":null,"B":null,"C":null}},{"key":"simplejev-qwen3.6-35b-a3b","display":"SimpleJev Qwen3.6-35B-A3B","addendum":null,"author":"Featherless AI","repo":"https://github.com/featherless-ai/simple-jev","class":"jev-rebuild","licence":"Apache-2.0 (Qwen weights); repository licence not stated","open":"yes","underlying":"Qwen3.6-35B-A3B through SimpleJev's direct-logit classifier","endpoint_kind":"demo","endpoint_condition":"the author's public demo endpoint","gpu":null,"adapter_id":"harness/specs/simplejev-qwen3.6-35b-a3b.json:jevbench.adapters.typesafe.TypeSafeAdapter","api_flag":true,"api_exposure_note":"API measurement: the operator's endpoint received sealed item text, without answers.","status":{"status":"partial","rows":678,"missing":946,"answered_ok":677},"full_coverage":true,"support":{"choice":"native","noul":"native","score":"native"},"planned_weight_denominator":1.0,"axes":{"intelligence":0.0,"calibration":78.49852428212563,"speed":75.18371345389433,"cost":35.1485010409676},"scores":{"A":0.0,"B":0.0,"C":0.0},"views":{"Balanced 33:33:33":0.0,"Emphasis on Accuracy 60:20:20":0.0,"Emphasis on Cost 20:20:60":0.0,"Emphasis on Speed 20:60:20":0.0,"Intelligence only":0.0},"composite_ci95":{"A":[0.0,0.0],"B":[0.0,0.0],"C":[0.0,0.0]},"jevbench_score":0.0,"intelligence":{"I_open":-17.361123164086187,"I_sealed":-22.966144581980604,"base":-20.163633873033397,"gap":5.605021417894417,"excess":0.4183939178151741,"penalty":1.0,"per_type_split":{"open|choice":{"cc":16.21556622973974,"tiers":{"easy":27.645462767561053,"standard":20.457499350142964,"judge":27.385892116182585,"hard":2.8593811202506667},"n":{"easy":72,"standard":84,"judge":114,"hard":169}},"open|noul":{"cc":-36.28602527092897,"tiers":{"easy":-47.05882352941176,"standard":-18.181818181818176,"judge":-23.529411764705888,"hard":-52.212389380530965},"n":{"easy":34,"standard":66,"judge":102,"hard":113}},"open|score":{"cc":-32.012910451069345,"tiers":{"easy":-61.6763097560751,"standard":-30.179560922548788,"judge":-26.90162155364877,"hard":-29.34720206214361},"n":{"easy":16,"standard":50,"judge":34,"hard":50}},"sealed|choice":{"cc":8.941634454184864,"tiers":{"easy":17.968749999999982,"standard":4.912836767036478,"judge":15.194346289752657,"hard":4.00972053462943},"n":{"easy":50,"standard":101,"judge":101,"hard":108}},"sealed|noul":{"cc":-42.690909090909095,"tiers":{"easy":-12.0,"standard":-48.0,"judge":-36.0,"hard":-52.727272727272734},"n":{"easy":25,"standard":50,"judge":50,"hard":55}},"sealed|score":{"cc":-35.149159109217585,"tiers":{"easy":-7.190320322368482,"standard":-36.15560725047939,"judge":-37.42010434672649,"hard":-39.93243580716728},"n":{"easy":25,"standard":50,"judge":50,"hard":55}}}},"validity":{"choice":{"n":799,"invalid_rate":0.5794743429286608},"noul":{"n":495,"invalid_rate":0.5717171717171717},"score":{"n":330,"invalid_rate":0.6090909090909091}},"calibration":{"score":78.49852428212563,"parts":{"choice":{"ece_hard":0.13284368943242192,"ece_score":73.43126211351561,"mean_tvd":0.3785573581540708,"n_tvd":19.0,"n":336.0,"n_ece_hard":103.0,"score":67.78776314905426},"noul":{"ece":0.06196859662122321,"brier":0.10697271930761937,"n":212.0,"score":87.60628067575536,"abstention_rate":0.21698113207547165},"score":{"nrps":0.08336163853547077,"top_ece":0.1573038905165916,"n":129.0,"interval90_coverage":0.8682170542635659,"interval90_width":0.2034883720930232,"score":80.1015290215673}}},"speed":{"p50_s_raw":0.8297080211341381,"p95_s_raw":0.9133629161864518,"p50_s_adjusted":1.6594160422682762,"p95_s_adjusted":1.8267258323729036,"n":198,"adjustment":"x2 demo-endpoint adjustment (assumption, not measured)"},"cost":{"kind":"estimate","usd_per_1000":0.1451164454277286,"basis":"ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)"},"rank":null,"ranked":false,"listing":"partial","not_ranked_because":"Partial run: 677 of 1,624 decisions answered; the missing ones count wrong and the row is not ranked.","ranks":{"A":null,"B":null,"C":null}}],"not_measured":[{"key":"jobe-qwen3.5-4b","display":"Jobe Qwen3.5-4B (frozen)","author":"MantisShrimpdev","addendum":null,"status":"not measured","reason":null},{"key":"mica-v01-4b","display":"mica-v01-4b","author":"unknown","addendum":{"id":"A1","release":"v1.5","label":"v1.5 roster addendum A1"},"status":"not measured","reason":null},{"key":"openjev-razorback16","display":"OpenJev (DiffusionGemma 26B-A4B NVFP4, razorback16)","author":"razorback16 / Codiv","addendum":null,"status":"not measured","reason":null}],"roster_count":103,"headline_hold":null}