{ "benchmark": "JevBench", "revision": "v1.6.0", "protocol": "jevbench::v1.6", "provisional": false, "status": "published", "run_kind": "scheduled-refresh", "label": "JevBench v1.6.0", "source_sha256": "04ad6eb3cfc06e78784a51629c8e2acab3e0acbe8134f3f00bd72bfcf61c949b", "source_note": "Full re-measure, score-overnight-12/out-O1S, scored 2026-10-05 09:06:16 UTC; O1S (method addendum B). Sage is a supplementary A3 measurement scored on 5 Oct 2026 with O1S and the strict full-run cost basis; its scorer source hash is recorded on its row.", "sample": { "open": 300, "sealed": 1200, "total": 1500, "published_open": 300 }, "types": { "choice": 0.3333333333333333, "noul": 0.3333333333333333, "score": 0.3333333333333333 }, "tier_weights": { "easy": 0.1, "hard": 0.4, "judge": 0.3, "standard": 0.2 }, "sealed_share_of_intelligence": 0.5, "G_med": 2.573387642438244, "G_med_api_basis_P_vs_A": -1.1922125858968342, "G_med_flag_gt10": false, "headline": "A", "options": { "A": { "intelligence_floor": 50.0, "weights": { "calibration": 25, "cost": 25, "intelligence": 25, "speed": 25 } }, "B": { "intelligence_floor": 50.0, "weights": { "calibration": 20, "cost": 20, "intelligence": 40, "speed": 20 } }, "C": { "intelligence_floor": 60.0, "weights": { "calibration": 25, "cost": 25, "intelligence": 25, "speed": 25 } } }, "views": [ "Balanced 33:33:33", "Emphasis on Accuracy 60:20:20", "Emphasis on Cost 20:20:60", "Emphasis on Speed 20:60:20", "Intelligence only" ], "bootstrap": { "B": 1000, "bootstrap_seed": 16, "g_med_fixed": true }, "n_ranked": 92, "board": { "A": { "order": [ "quyet-1-0-large", "jev-1.13.0", "wity-1", "torchcast-decision-12b", "deck31b", "winnow-12b", "cygnet", "jev-omni", "sage-1.3.0", "plumb-4b", "spark-s1-4b-v6", "swanone", "quyet-1-0-medium", "lev", "ninfer-qwen3.8-flash-next", "jev-local", "decider-4b-v2", "jevk5-v0.3-4b", "metask-jev-4b", "deck-4b-v1-0", "clef-flash", "qwen35-9b-jev-data-mix-v2", "jevk5-v02", "jevone", "hopper", "imajev-4b-rtx5090-a2", "malkuth-4b", "jqv", "decider-35b-a3b", "manchego-v21-v15", "decision-4b-v12", "raw-qwen3-4b-instruct-2507", "nimble-9b", "reflex-4b", "typecastlm", "decision-4b-v11", "autojev-27b", "eikos-27b", "ninfer-qwen3.8-27b", "openjev-thinking", "open-jev-zefan-9b", "standardone-8b", "clef", "jev-qwen3.5-9b-base-nvfp4", "semif-qwen3.5-4b", "bev-bonsai-27b", "litjev", "localjev-qwen3.5-4b", "system-one-sg", "kev-8b", "decision-2b", "opensourcejev-qwen35-4b-q4km", "open-alternative-jev", "raw-qwen3-8b", "decider-2b", "nemotron-diffusion-8b", "kev-4b", "malkuth-2b", "qwen3-reranker-4b", "zerank-2", "open-jev-zefan-2b", "open-jev-zefan-27b-v1.1", "raw-phi-4-mini", "simplejev-qwen3.5-0.8b", "smalljev", "kev-0.6b", "opendecision", "mghafiri-qwen3.5-0.8b-decision-model", "deem-0.8-v1", "decision-fast", "quyet-1-0-small-en", "fastino-gliner-2-5-decide", "laya-typed-decisions", "openjev-verdict-1.4", "raw-qwen3-0.6b", "lev-350m", "raw-qwen3-1.7b", "kev-0.5b", "openjev-verdict", "quyet-1-0-small", "quyet-1-0-tiny", "open-jev-deberta-v3-large", "verdict-small", "laya", "mxbai-rerank-base-v2", "laya-multilingual", "certo", "clm-8b", "bge-reranker-v2-m3", "gte-reranker-modernbert-base", "mirror", "open-jev-json-canvas-joshuasp" ], "leader_wording": null, "markers": [] }, "B": { "order": [ "quyet-1-0-large", "wity-1", "deck31b", "jev-1.13.0", "torchcast-decision-12b", "winnow-12b", "cygnet", "jev-omni", "sage-1.3.0", "plumb-4b", "swanone", "spark-s1-4b-v6", "ninfer-qwen3.8-flash-next", "quyet-1-0-medium", "lev", "jev-local", "decider-4b-v2", "clef-flash", "metask-jev-4b", "deck-4b-v1-0", "jevk5-v0.3-4b", "qwen35-9b-jev-data-mix-v2", "jevone", "jevk5-v02", "hopper", "imajev-4b-rtx5090-a2", "decider-35b-a3b", "malkuth-4b", "jqv", "manchego-v21-v15", "decision-4b-v12", "nimble-9b", "raw-qwen3-4b-instruct-2507", "eikos-27b", "autojev-27b", "openjev-thinking", "reflex-4b", "ninfer-qwen3.8-27b", "clef", "typecastlm", "open-jev-zefan-9b", "decision-4b-v11", "standardone-8b", "jev-qwen3.5-9b-base-nvfp4", "semif-qwen3.5-4b", "bev-bonsai-27b", "litjev", "system-one-sg", "localjev-qwen3.5-4b", "kev-8b", "opensourcejev-qwen35-4b-q4km", "raw-qwen3-8b", "decision-2b", "open-alternative-jev", "decider-2b", "nemotron-diffusion-8b", "kev-4b", "open-jev-zefan-27b-v1.1", "malkuth-2b", "qwen3-reranker-4b", "open-jev-zefan-2b", "zerank-2", "raw-phi-4-mini", "simplejev-qwen3.5-0.8b", "smalljev", "kev-0.6b", "opendecision", "mghafiri-qwen3.5-0.8b-decision-model", "deem-0.8-v1", "decision-fast", "fastino-gliner-2-5-decide", "quyet-1-0-small-en", "laya-typed-decisions", "raw-qwen3-0.6b", "openjev-verdict-1.4", "raw-qwen3-1.7b", "lev-350m", "kev-0.5b", "openjev-verdict", "quyet-1-0-small", "quyet-1-0-tiny", "open-jev-deberta-v3-large", "verdict-small", "laya", "mxbai-rerank-base-v2", "laya-multilingual", "certo", "clm-8b", "bge-reranker-v2-m3", "gte-reranker-modernbert-base", "mirror", "open-jev-json-canvas-joshuasp" ], "leader_wording": null, "markers": [] }, "C": { "order": [ "quyet-1-0-large", "jev-1.13.0", "wity-1", "torchcast-decision-12b", "deck31b", "winnow-12b", "jev-omni", "cygnet", "sage-1.3.0", "swanone", "plumb-4b", "spark-s1-4b-v6", "quyet-1-0-medium", "lev", "ninfer-qwen3.8-flash-next", "jev-local", "decider-4b-v2", "jevk5-v0.3-4b", "metask-jev-4b", "deck-4b-v1-0", "clef-flash", "qwen35-9b-jev-data-mix-v2", "jevk5-v02", "jevone", "hopper", "imajev-4b-rtx5090-a2", "malkuth-4b", "eikos-27b", "jqv", "openjev-thinking", "decider-35b-a3b", "manchego-v21-v15", "decision-4b-v12", "clef", "autojev-27b", "raw-qwen3-4b-instruct-2507", "nimble-9b", "reflex-4b", "typecastlm", "ninfer-qwen3.8-27b", "decision-4b-v11", "open-jev-zefan-9b", "standardone-8b", "jev-qwen3.5-9b-base-nvfp4", "semif-qwen3.5-4b", "bev-bonsai-27b", "litjev", "localjev-qwen3.5-4b", "system-one-sg", "kev-8b", "decision-2b", "opensourcejev-qwen35-4b-q4km", "open-alternative-jev", "raw-qwen3-8b", "decider-2b", "nemotron-diffusion-8b", "kev-4b", "malkuth-2b", "qwen3-reranker-4b", "zerank-2", "open-jev-zefan-2b", "open-jev-zefan-27b-v1.1", "raw-phi-4-mini", "simplejev-qwen3.5-0.8b", "smalljev", "kev-0.6b", "opendecision", "mghafiri-qwen3.5-0.8b-decision-model", "deem-0.8-v1", "decision-fast", "quyet-1-0-small-en", "fastino-gliner-2-5-decide", "laya-typed-decisions", "openjev-verdict-1.4", "raw-qwen3-0.6b", "lev-350m", "raw-qwen3-1.7b", "kev-0.5b", "openjev-verdict", "quyet-1-0-small", "quyet-1-0-tiny", "open-jev-deberta-v3-large", "verdict-small", "laya", "mxbai-rerank-base-v2", "laya-multilingual", "certo", "clm-8b", "bge-reranker-v2-m3", "gte-reranker-modernbert-base", "mirror", "open-jev-json-canvas-joshuasp" ], "leader_wording": null, "markers": [] } }, "v16": { "a2_supplement": { "counts": { "A": 300, "P": 300, "S": 300, "api_input": 600, "api_smoke_public": 5, "core": 0, "selfhosted_input": 600 }, "release": "v1.6.0-A2", "set_manifest_sha256": { "A": "380f76186ec6e09130154579c6ae1db0eebd1cae1b219339ad7429458c177f5d", "P": "6b2321f8359d87769a95c8e52135542d95ccdc353197e1e5c96abb9b616ce5ba", "S": "380f76186ec6e09130154579c6ae1db0eebd1cae1b219339ad7429458c177f5d" }, "supplement_of": "v1.6.0" }, "capability_formula": "mean(Intelligence, Calibration)", "counts": { "A": 300, "P": 300, "S": 1200, "api_input": 600, "api_smoke_public": 5, "core": 900, "selfhosted_input": 1500 }, "equating": { "equating_flag": null, "min_pool": 5, "offsets": { "C": 2.3428887908654588, "I": -2.209917383979878 }, "rule": "median over complete full-coverage self-hosted systems of metric(S u P) - metric(A u P), per metric (Intelligence, Calibration); API value = clip(raw_A + offset)", "pool_note": "Median over the complete full-coverage self-hosted systems of this run (at least five required)." }, "equating_A2": { "G_med_A2_basis_P_vs_A2": 3.824932047119341, "diagnostic_main_A_offset_same_systems": { "C": 1.9486117388517954, "I": -4.381056886943014 }, "equating_flag": null, "offset_bootstrap_B": 200, "offsets": { "C": 1.5638258868659562, "I": -0.2887166717454388 }, "per_system": { "hopper": { "C_A2": 87.82068552843725, "C_A_main": 87.40904942545944, "C_S": 90.07902805935959, "I_A2": 34.77120638990772, "I_A_main": 41.88389851544817, "I_S": 34.73970870443837, "dC": 2.2583425309223344, "dI": -0.03149768546934695 }, "kev-0.6b": { "C_A2": 68.46161035185293, "C_A_main": 65.17717796738081, "C_S": 66.26655558235618, "I_A2": 5.866380901403119, "I_A_main": 9.100409843273326, "I_S": 7.598789169415934, "dC": -2.1950547694967497, "dI": 1.732408268012815 }, "kev-4b": { "C_A2": 68.65918994957973, "C_A_main": 69.97372951951687, "C_S": 68.45285936403559, "I_A2": 19.558821195537895, "I_A_main": 25.023085194237723, "I_S": 19.510707720813777, "dC": -0.20633058554413708, "dI": -0.04811347472411853 }, "kev-8b": { "C_A2": 61.3782880436933, "C_A_main": 67.24332350856244, "C_S": 70.7561729087212, "I_A2": 26.42642690493986, "I_A_main": 34.2874494947909, "I_S": 29.26217941280341, "dC": 9.377884865027895, "dI": 2.8357525078635497 }, "localjev-qwen3.5-4b": { "C_A2": 76.77849725935185, "C_A_main": 81.40355743693019, "C_S": 85.84286362718517, "I_A2": 22.674863142962646, "I_A_main": 28.26930138473988, "I_S": 24.145363218466134, "dC": 9.064366367833316, "dI": 1.4705000755034874 }, "malkuth-4b": { "C_A2": 83.46678991361165, "C_A_main": 85.35554104202156, "C_S": 88.43741591513763, "I_A2": 34.73305384360792, "I_A_main": 38.819192380023466, "I_S": 33.840703805987175, "dC": 4.970626001525986, "dI": -0.8923500376207443 }, "metask-jev-4b": { "C_A2": 79.1415556283946, "C_A_main": 83.18850868556241, "C_S": 84.41575352936586, "I_A2": 41.051921128504205, "I_A_main": 41.86894313160032, "I_S": 39.166454612935595, "dC": 5.274197900971259, "dI": -1.8854665155686092 }, "raw-phi-4-mini": { "C_A2": 56.153985406303406, "C_A_main": 53.16092441580774, "C_S": 57.015465878675315, "I_A2": 14.189557846849354, "I_A_main": 12.927281214813327, "I_S": 13.660237978082595, "dC": 0.861480472371909, "dI": -0.5293198687667591 }, "raw-qwen3-1.7b": { "C_A2": 11.425912172792259, "C_A_main": 11.45520847967844, "C_S": 11.516326812547591, "I_A2": 4.994829708920566, "I_A_main": 6.997731901083261, "I_S": 5.124286554530066, "dC": 0.09041463975533226, "dI": 0.1294568456095 }, "raw-qwen3-4b-instruct-2507": { "C_A2": 34.31244267053921, "C_A_main": 37.75992930947443, "C_S": 35.18175191334879, "I_A2": 35.69189443401588, "I_A_main": 38.58935302861775, "I_S": 33.95117742100547, "dC": 0.8693092428095781, "dI": -1.7407170130104106 }, "raw-qwen3-8b": { "C_A2": 34.345565335170505, "C_A_main": 38.60246474366818, "C_S": 33.721548452619835, "I_A2": 30.34325924789806, "I_A_main": 31.73129272246627, "I_S": 26.54912920590792, "dC": -0.62401688255067, "dI": -3.7941300419901403 }, "typecastlm": { "C_A2": 76.627802179825, "C_A_main": 79.3986887571586, "C_S": 83.29951051808085, "I_A2": 31.896472425382363, "I_A_main": 31.998923946026117, "I_S": 29.78900656204624, "dC": 6.671708338255854, "dI": -2.107465863336124 } }, "pool": [ "hopper", "kev-0.6b", "kev-4b", "kev-8b", "localjev-qwen3.5-4b", "malkuth-4b", "metask-jev-4b", "raw-phi-4-mini", "raw-qwen3-1.7b", "raw-qwen3-4b-instruct-2507", "raw-qwen3-8b", "typecastlm" ], "pool_n": 12, "rule": "median over the A2 pool of metric(S u P, main run) - metric(A2 u P, A2 run), per metric; API A2 value = clip(raw_A2 + offset_A2)", "runs_status": { "hopper": { "answered_ok": 600, "missing": 0, "rows": 600, "status": "complete" }, "kev-0.6b": { "answered_ok": 594, "missing": 0, "rows": 600, "status": "complete" }, "kev-4b": { "answered_ok": 594, "missing": 0, "rows": 600, "status": "complete" }, "kev-8b": { "answered_ok": 594, "missing": 0, "rows": 600, "status": "complete" }, "localjev-qwen3.5-4b": { "answered_ok": 600, "missing": 0, "rows": 600, "status": "complete" }, "malkuth-4b": { "answered_ok": 594, "missing": 0, "rows": 600, "status": "complete" }, "metask-jev-4b": { "answered_ok": 594, "missing": 0, "rows": 600, "status": "complete" }, "raw-phi-4-mini": { "answered_ok": 594, "missing": 0, "rows": 600, "status": "complete" }, "raw-qwen3-1.7b": { "answered_ok": 594, "missing": 0, "rows": 600, "status": "complete" }, "raw-qwen3-4b-instruct-2507": { "answered_ok": 594, "missing": 0, "rows": 600, "status": "complete" }, "raw-qwen3-8b": { "answered_ok": 594, "missing": 0, "rows": 600, "status": "complete" }, "typecastlm": { "answered_ok": 600, "missing": 0, "rows": 600, "status": "complete" } } }, "g_med_rule": "median gap over ranked self-hosted systems; API systems use the same systems' P-vs-A gap", "offset_bootstrap_B": 200, "release": "v1.6.0", "scorer_sources_sha256": { "noul_method_v16.py": "119e59486ab3300b972fbcbadb0fe1dceceb124287607375aa61c66063c607d0", "score_v15.py": "0931e05c6410b1b971201834397cb609cdb25f942723c50375632952f6dcc447", "score_v15_headlineA.py": "826f2cbcb57007ec8a161e6eb0111182b8a656c0638027c6c78022a858fd8e2c", "score_v16.py": "a634ba2b66883091bba0cb1df925f6fb4c487c73210d990a70186a02f80f1b7b" }, "scoring_parameters": { "B": 1000, "g_med_fixed": null, "noul_method": "O1S", "only": null, "seed": 16, "test_allow_partial": false, "workers": 10 }, "set_manifest_sha256": { "A": "19b196a71d58da260cd83069ffcd64bb323e158c24a5fcabeffbb5bee7c1eabb", "P": "6b2321f8359d87769a95c8e52135542d95ccdc353197e1e5c96abb9b616ce5ba", "S": "f13557a141fba12efc6bf6d39a0dd8e3bf803d4cacd6eab516d4f4f87045539a" }, "long_items_note": "23 items of this draw are very long (about 77,000 to 81,000 input tokens). Systems with a shorter context window refuse them; Jev-Omni runs out of GPU memory on them on its listed RTX 6000 (48 GB) recipe; decider-4b v2's server truncates them to about 32,800 tokens and answers; Plumb-4B reads them in full. Each outcome is the system's own recipe and is scored as such.", "item_sets": [ { "set": "S", "name": "Sealed release draw", "items": 1200, "by_type": { "choice": 600, "noul": 300, "score": 300 }, "answered_by": "self-hosted systems only (run offline on our own GPU pods or Sandy)" }, { "set": "A", "name": "API subset (part of S)", "items": 300, "by_type": { "choice": 150, "noul": 75, "score": 75 }, "answered_by": "self-hosted systems and externally hosted APIs" }, { "set": "P", "name": "Public set", "items": 300, "by_type": { "choice": 150, "noul": 75, "score": 75 }, "answered_by": "every system" } ] }, "noul_method": { "applied": "O1S", "decisive_source_sha256": "84cfc6ae77fe9f37272b9a038187023614a242cbc83e782608c4479aeb36acc1" }, "systems": [ { "addendum": null, "api_flag": false, "api_subset": false, "author": "Chinh Nguyen (ncchinh)", "axes": { "calibration": 89.97624968733336, "cost": 50.546097335562465, "intelligence": 73.40135940607429, "speed": 86.87440406834797 }, "calibration": { "parts": { "choice": { "ece_hard": 0.05296499999999975, "ece_score": 89.40700000000005, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 89.40700000000005 }, "noul": { "abstention_rate": 0.08799999999999997, "brier": 0.07739600085333337, "ece": 0.046341333333333186, "n": 375, "score": 90.73173333333337 }, "score": { "interval90_coverage": 0.9413333333333334, "interval90_width": 0.21599999999999997, "n": 375, "nrps": 0.054102618759999964, "score": 89.79001572866669, "top_ece": 0.07504853333333314 } }, "score": 89.97624968733336 }, "capability": 81.68880454670384, "composite_ci95": { "A": [ 69.83728305350246, 72.20621881721509 ], "B": [ 69.73317863481549, 73.04885527628076 ], "C": [ 69.83728305350246, 72.20621881721509 ] }, "cost": { "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): OpenRouter google/gemma-4-31b-it list price $0.09/M input, $0.34/M output (the exact base model google/gemma-4-31B-it; listed since Apr 2026), read 2026-10-04 x the package's own measured usage.input_tokens; one forward pass, nothing generated; run in process on 1x H100 80GB (Lium), quyet 1.0.0; estimated, not charged", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.04451059729064039 }, "display": "Quyet-1.0-Large (Chinh Nguyen, Gemma-4-31B decoder, option-letter logits)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 75.23869318243646, "I_sealed": 71.56402562971212, "base": 73.40135940607429, "excess": 1.1012799102861002, "gap": 3.674667552724344, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 85.01378955328916, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 100.00000000000001, "hard": 84.90091223655237, "judge": 80.22412656558997, "standard": 84.93093344495604 } }, "open|noul": { "cc": 71.3768115942029, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 82.6086956521739, "judge": 50.0, "standard": 66.66666666666667 } }, "open|score": { "cc": 69.32547839981737, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 57.654545454545456, "hard": 72.09425925925925, "judge": 68.68847457627118, "standard": 70.5788888888889 } }, "sealed|choice": { "cc": 82.59643939217574, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 81.37609932747026, "hard": 80.65610859728505, "judge": 81.3780260707635, "standard": 88.91489099642814 } }, "sealed|noul": { "cc": 66.16688233643043, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 84.00000000000001, "hard": 45.63106796116505, "judge": 76.3157894736842, "standard": 83.09859154929578 } }, "sealed|score": { "cc": 65.9287551605302, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 73.00248226950355, "hard": 67.1347549019608, "judge": 65.63335329341317, "standard": 60.42299492385786 } } } }, "jevbench_score": 71.38706329141472, "key": "quyet-1-0-large", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 1, "ranked": true, "ranks": { "A": 1, "B": 1, "C": 1, "capability": 1 }, "scores": { "A": 71.38706329141472, "B": 71.78102942459306, "C": 71.38706329141472 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3815564611926675, "p50_s_raw": 0.11577823059633374, "p95_s_adjusted": 0.5382707504555583, "p95_s_raw": 0.19413537522777913 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 85.58390587934255, 91.370785899262 ], "I": [ 69.18945219901103, 76.93636317458929 ], "capability": [ 77.9843219931321, 83.55754626687737 ], "composite": { "A": [ 69.83728305350246, 72.20621881721509 ], "B": [ 69.73317863481549, 73.04885527628076 ], "C": [ 69.83728305350246, 72.20621881721509 ] }, "seed": 86 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.97624968733336, "capability": 81.68880454670384, "intelligence": 73.40135940607429 }, "per_type": { "choice": { "accuracy": 0.8706666666666667, "calibration": 89.40700000000005, "cc": 83.46011386565368, "cc_by_type": { "choice": 83.46011386565368 }, "cc_tiered": 83.80511447273244, "cc_tiered_open": 85.01378955328916, "cc_tiered_sealed": 82.59643939217574, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 83.46011386565368 }, "noul": { "accuracy": 0.848, "calibration": 90.73173333333337, "cc": 69.6, "cc_by_type": { "noul": 69.6 }, "cc_tiered": 68.77184696531667, "cc_tiered_open": 71.3768115942029, "cc_tiered_sealed": 66.16688233643043, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 69.6 }, "score": { "accuracy": 0.7493333333333333, "calibration": 89.79001572866669, "cc": 66.83812280701756, "cc_by_type": { "score": 66.83812280701756 }, "cc_tiered": 67.6271167801738, "cc_tiered_open": 69.32547839981737, "cc_tiered_sealed": 65.9287551605302, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 66.83812280701756 } }, "raw_on_A": { "calibration": 86.55147389375357, "capability": 81.17122174668583, "intelligence": 75.7909695996181 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 66.78760078772265, "Emphasis on Accuracy 60:20:20": 69.28473442746802, "Emphasis on Cost 20:20:60": 59.18115107083417, "Emphasis on Speed 20:60:20": 73.59406855476429, "Intelligence only": 73.40135940607429 }, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.912, "acc_among_decisive": 0.9298245614035088, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "author submission" }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Large", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (order 3643732b)", "underlying": "google/gemma-4-31B-it" }, { "addendum": null, "api_flag": true, "api_subset": true, "author": "TypeSafe AI", "axes": { "calibration": 90.56678401753481, "cost": 54.72500234484654, "intelligence": 62.40124449091305, "speed": 91.49989299046987 }, "calibration": { "parts": { "choice": { "ece_hard": 0.06289890777695657, "ece_score": 87.4202184446087, "mean_tvd": null, "n": 293, "n_ece_hard": 123, "n_tvd": 0, "score": 87.4202184446087 }, "noul": { "abstention_rate": 0.17333333333333334, "brier": 0.06017133333333333, "ece": 0.0599333333333333, "n": 150, "score": 88.01333333333335 }, "score": { "interval90_coverage": 0.9523809523809523, "interval90_width": 0.2335600907029478, "n": 147, "nrps": 0.05879122836972765, "score": 89.23813390206607, "top_ece": 0.07822304679447543 } }, "score": 88.22389522666936 }, "capability": 76.48401425422394, "composite_ci95": { "A": [ 66.98534594258273, 72.53033754680197 ], "B": [ 63.71843377650008, 71.60515813438901 ], "C": [ 53.12871127236506, 72.53033754680197 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): operator standard launch list price (interpretation I-1); no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.032297327586206896 }, "display": "Jev 1.13.0 (TypeSafe AI)", "endpoint_kind": "api", "full_coverage": true, "intelligence": { "I_open": 65.60411433522728, "I_sealed": 63.61820941455857, "base": 64.61116187489293, "excess": 3.178117506565543, "gap": 1.985904920668709, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 79.0195653974535, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 71.68921044353571, "judge": 84.17930125247197, "standard": 77.39640016743407 } }, "open|noul": { "cc": 54.46376811594203, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 47.82608695652173, "judge": 39.99999999999999, "standard": 66.66666666666667 } }, "open|score": { "cc": 63.32900949228633, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 76.45454545454547, "hard": 65.44837261503929, "judge": 68.91525423728812, "standard": 44.148148148148145 } }, "sealed|choice": { "cc": 72.90953748043711, "n": { "easy": 21, "hard": 62, "judge": 42, "standard": 25 }, "tiers": { "easy": 87.97996661101837, "hard": 58.69191049913941, "judge": 82.02396804260984, "standard": 80.13793103448276 } }, "sealed|noul": { "cc": 59.6027149321267, "n": { "easy": 13, "hard": 25, "judge": 20, "standard": 17 }, "tiers": { "easy": 84.61538461538463, "hard": 28.000000000000004, "judge": 89.99999999999999, "standard": 64.70588235294117 } }, "sealed|score": { "cc": 58.34237583111192, "n": { "easy": 13, "hard": 33, "judge": 10, "standard": 19 }, "tiers": { "easy": 80.8695652173913, "hard": 32.251233262861156, "judge": 95.25581395348837, "standard": 43.89090909090909 } } } }, "jevbench_score": 71.08703004565037, "key": "jev-1.13.0", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 2, "ranked": true, "ranks": { "A": 2, "B": 4, "C": 2, "capability": 6 }, "scores": { "A": 71.08703004565037, "B": 69.16167274556517, "C": 71.08703004565037 }, "speed": { "adjustment": "none (API)", "n": 124, "p50_s_adjusted": 0.23910215869545937, "p50_s_raw": 0.23910215869545937, "p95_s_adjusted": 0.29609236158430574, "p95_s_raw": 0.29609236158430574 }, "status": { "answered_ok": 590, "missing": 0, "rows": 600, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": true, "api_subset_tag": "A", "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 83.10550154095336, 91.7935013819524 ], "I": [ 53.33245232101987, 68.06807165018392 ], "capability": [ 69.1907647929543, 78.7556482385829 ], "composite": { "A": [ 66.98534594258273, 72.53033754680197 ], "B": [ 63.71843377650008, 71.60515813438901 ], "C": [ 53.12871127236506, 72.53033754680197 ] }, "seed": 44 }, "ci95_basis": "bootstrap over the API system's items (n = A sealed items) AND over the equating offset (pool systems resampled with replacement, items resampled; B_offset = 200)", "ci95_n": 300, "complete": true, "equated": { "calibration": 90.56678401753481, "capability": 76.48401425422394, "intelligence": 62.40124449091305 }, "equating_flag": null, "equating_offset": { "C": 2.3428887908654588, "I": -2.209917383979878 }, "g_med_used": -1.1922125858968342, "lane": "api", "metric_basis": "A u P (sealed side = A, n=300)", "n_items": 600, "n_public": 300, "n_sealed_side": 300, "offset_ci95": { "C": [ 1.8692848511563234, 5.1754496498420055 ], "I": [ -4.708219277619042, -0.4755385207395287 ] }, "on_S": null, "per_type": { "choice": { "accuracy": 0.8166666666666667, "calibration": 87.4202184446087, "cc": 76.67569796206854, "cc_by_type": { "choice": 76.67569796206854 }, "cc_tiered": 75.9645514389453, "cc_tiered_open": 79.0195653974535, "cc_tiered_sealed": 72.90953748043711, "low_n": false, "n": 300, "n_by_type": { "choice": 300 }, "score": 76.67569796206854 }, "noul": { "accuracy": 0.8066666666666666, "calibration": 88.01333333333335, "cc": 61.33333333333333, "cc_by_type": { "noul": 61.33333333333333 }, "cc_tiered": 57.033241524034366, "cc_tiered_open": 54.46376811594203, "cc_tiered_sealed": 59.6027149321267, "low_n": false, "n": 150, "n_by_type": { "noul": 150 }, "score": 61.33333333333333 }, "score": { "accuracy": 0.74, "calibration": 89.23813390206607, "cc": 60.94754989722164, "cc_by_type": { "score": 60.94754989722164 }, "cc_tiered": 60.83569266169913, "cc_tiered_open": 63.32900949228633, "cc_tiered_sealed": 58.34237583111192, "low_n": false, "n": 150, "n_by_type": { "score": 150 }, "score": 60.94754989722164 } }, "raw_on_A": { "calibration": 88.22389522666936, "capability": 76.41752855078114, "intelligence": 64.61116187489293 }, "set": "A+P" }, "validity": { "choice": { "invalid_rate": 0.023333333333333317, "n": 300 }, "noul": { "invalid_rate": 0.0, "n": 150 }, "score": { "invalid_rate": 0.020000000000000018, "n": 150 } }, "views": { "Balanced 33:33:33": 66.33135537134838, "Emphasis on Accuracy 60:20:20": 64.70136519155085, "Emphasis on Cost 20:20:60": 61.14424415831871, "Emphasis on Speed 20:60:20": 74.53183387901865, "Intelligence only": 62.40124449091305 }, "class": "jev", "open": "no", "licence": "proprietary API", "repo": "https://docs.typesafe.ai", "gpu": null, "endpoint_condition": "the operator's hosted API", "underlying": "closed", "adapter_id": "harness/specs/jev-1.13.0.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "model_pin": "jev-1.13.0 (response model id on all 590 successful rows)", "last_measured_on": "2026-10-01", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.8266666666666667, "acc_among_decisive": 0.9758064516129032, "valid": 150 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": true, "api_subset": true, "author": "Wity (alphanimble)", "axes": { "calibration": 87.97057132793138, "cost": 58.368182152360596, "intelligence": 71.55290364428411, "speed": 71.79000307789984 }, "calibration": { "parts": { "choice": { "ece_hard": 0.09949788321926671, "ece_score": 80.10042335614665, "mean_tvd": null, "n": 300, "n_ece_hard": 128, "n_tvd": 0, "score": 80.10042335614665 }, "noul": { "abstention_rate": 0.09999999999999998, "brier": 0.08507127503806897, "ece": 0.06456836759830527, "n": 150, "score": 87.08632648033895 }, "score": { "interval90_coverage": 0.9666666666666667, "interval90_width": 0.19777777777777775, "n": 150, "nrps": 0.05061731342287607, "score": 92.03348648671069, "top_ece": 0.05435647842145506 } }, "score": 86.40674544106542 }, "capability": 79.76173748610775, "composite_ci95": { "A": [ 68.00204720585086, 73.31665078356933 ], "B": [ 67.16737616420112, 74.18796382164403 ], "C": [ 68.00204720585086, 73.31665078356933 ] }, "cost": { "basis": "Wity's stated API tariff: USD 0.042 per 1M input tokens, output free; v1.5 cost per 1,000 decisions carried from the v1.5.8 build (measured 944,200 input tokens over 1,624 decisions, build f65ce826a455). API models with a known base are ranked at their own API price (1 Oct 2026 rule).", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.02441896551724138 }, "display": "wity-1 (Wity, reasoning auto)", "endpoint_kind": "api", "full_coverage": true, "intelligence": { "I_open": 68.30373826167562, "I_sealed": 75.37950237038348, "base": 71.84162031602955, "excess": -10.900696155827198, "gap": -7.075764108707858, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 82.78833309663467, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 73.57659641396666, "judge": 84.17930125247197, "standard": 92.46546672247803 } }, "open|noul": { "cc": 58.2415458937198, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 47.82608695652173, "judge": 60.00000000000001, "standard": 55.55555555555556 } }, "open|score": { "cc": 63.881335794672395, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 81.26135069496806, "hard": 66.56206622564852, "judge": 62.421718489672884, "standard": 52.01929344007156 } }, "sealed|choice": { "cc": 80.68604951876662, "n": { "easy": 21, "hard": 60, "judge": 34, "standard": 35 }, "tiers": { "easy": 87.77589134125635, "hard": 72.37960339943344, "judge": 88.65307837781046, "standard": 81.80347755762232 } }, "sealed|noul": { "cc": 71.50058769753167, "n": { "easy": 13, "hard": 31, "judge": 12, "standard": 19 }, "tiers": { "easy": 84.61538461538463, "hard": 48.38709677419355, "judge": 100.0, "standard": 68.42105263157893 } }, "sealed|score": { "cc": 73.95186989485214, "n": { "easy": 12, "hard": 31, "judge": 12, "standard": 20 }, "tiers": { "easy": 77.05232185854771, "hard": 68.57225500280248, "judge": 75.38096803223011, "standard": 81.0172264910367 } } } }, "jevbench_score": 70.91538581175126, "key": "wity-1", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 3, "ranked": true, "ranks": { "A": 3, "B": 2, "C": 3, "capability": 2 }, "scores": { "A": 70.91538581175126, "B": 71.04197893690481, "C": 70.91538581175126 }, "speed": { "adjustment": "none (API)", "n": 124, "p50_s_adjusted": 1.5746641159057617, "p50_s_raw": 1.5746641159057617, "p95_s_adjusted": 4.205443101748823, "p95_s_raw": 4.205443101748823 }, "status": { "answered_ok": 600, "missing": 0, "rows": 600, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": true, "api_subset_tag": "A2", "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 80.26726502396788, 95.55467774882284 ], "I": [ 63.37800402943055, 78.33734855892178 ], "capability": [ 73.19833808279803, 86.02991816084987 ], "composite": { "A": [ 68.00204720585086, 73.31665078356933 ], "B": [ 67.16737616420112, 74.18796382164403 ], "C": [ 68.00204720585086, 73.31665078356933 ] }, "seed": 112 }, "ci95_basis": "bootstrap over the API system's items (n = A sealed items) AND over the equating offset (pool systems resampled with replacement, items resampled; B_offset = 200)", "ci95_n": 300, "complete": true, "equated": { "calibration": 87.97057132793138, "capability": 79.76173748610775, "intelligence": 71.55290364428411 }, "equating_flag": null, "equating_offset": { "C": 1.5638258868659562, "I": -0.2887166717454388 }, "g_med_used": 3.824932047119341, "lane": "api", "metric_basis": "A2 u P (sealed side = supplementary API subset A2, n=300)", "n_items": 600, "n_public": 300, "n_sealed_side": 300, "offset_ci95": { "C": [ -1.2071304620902201, 9.442982800888801 ], "I": [ -5.855912654795828, 4.37048407464939 ] }, "on_S": null, "per_type": { "choice": { "accuracy": 0.8566666666666667, "calibration": 80.10042335614665, "cc": 81.66919287608943, "cc_by_type": { "choice": 81.66919287608943 }, "cc_tiered": 81.73719130770064, "cc_tiered_open": 82.78833309663467, "cc_tiered_sealed": 80.68604951876662, "low_n": false, "n": 300, "n_by_type": { "choice": 300 }, "score": 81.66919287608943 }, "noul": { "accuracy": 0.8266666666666667, "calibration": 87.08632648033895, "cc": 65.33333333333333, "cc_by_type": { "noul": 65.33333333333333 }, "cc_tiered": 64.87106679562574, "cc_tiered_open": 58.2415458937198, "cc_tiered_sealed": 71.50058769753167, "low_n": false, "n": 150, "n_by_type": { "noul": 150 }, "score": 65.33333333333333 }, "score": { "accuracy": 0.8066666666666666, "calibration": 92.03348648671069, "cc": 70.01279544782105, "cc_by_type": { "score": 70.01279544782105 }, "cc_tiered": 68.91660284476227, "cc_tiered_open": 63.881335794672395, "cc_tiered_sealed": 73.95186989485214, "low_n": false, "n": 150, "n_by_type": { "score": 150 }, "score": 70.01279544782105 } }, "raw_on_A": { "calibration": 86.40674544106542, "capability": 79.12418287854749, "intelligence": 71.84162031602955 }, "set": "A2+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 300 }, "noul": { "invalid_rate": 0.0, "n": 150 }, "score": { "invalid_rate": 0.0, "n": 150 } }, "views": { "Balanced 33:33:33": 66.61069702768683, "Emphasis on Accuracy 60:20:20": 68.50332510768314, "Emphasis on Cost 20:20:60": 63.04927329412732, "Emphasis on Speed 20:60:20": 68.59007565188263, "Intelligence only": 71.55290364428411 }, "model_pin": null, "last_measured_on": null, "measurement_date_status": "unknown", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.9, "acc_among_decisive": 0.9185185185185185, "valid": 150 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "author submission" }, "class": "decision-api", "open": "no", "licence": "proprietary hosted API", "repo": "https://wity.alphanimble.com/", "endpoint_condition": "operator's hosted production API, measured as is; API default is off, our request sets reasoning=auto as the operator asked for their main row.", "underlying": "Qwen3.6-35B-A3B (operator self-reported; not independently verified)", "adapter_id": "run/wity-1-auto.spec.json:wity_reasoning_adapter.WityReasoningAdapter", "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "alt": { "label": "Qwen3.6-35B-A3B base-model reference price", "note": "Operator self-reported base, not independently verified; frozen Qwen/Qwen3.6-35B-A3B reference USD 0.15/M input and 1.00/M output, applied to the same measured tokens.", "usd_per_1000": 0.08721059113300492, "axes": { "calibration": 90.45687167393245, "cost": 41.78292309262718, "intelligence": 78.56795761907267, "speed": 75.05991709119236 } } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Torchcast AI", "axes": { "calibration": 82.93469131589123, "cost": 56.36034169226667, "intelligence": 60.54320091058725, "speed": 91.74573681567192 }, "calibration": { "parts": { "choice": { "ece_hard": 0.07394265024295099, "ece_score": 85.2114699514098, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 85.2114699514098 }, "noul": { "abstention_rate": 0.035040431266846306, "brier": 0.1393319820198383, "ece": 0.13485522993675053, "n": 371, "score": 73.02895401264989 }, "score": { "interval90_coverage": 0.964769647696477, "interval90_width": 0.33220415537488723, "n": 369, "nrps": 0.06922052747100878, "score": 90.56364998361398, "top_ece": 0.05975323642835579 } }, "score": 82.93469131589123 }, "capability": 71.73894611323924, "composite_ci95": { "A": [ 67.22388654747238, 71.4745219700647 ], "B": [ 64.35805126511531, 70.01822593897899 ], "C": [ 56.59214410663297, 71.4745219700647 ] }, "cost": { "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; no public tariff, customer asked for the base-model reference): OpenRouter google/gemma-3-12b-it list price $0.05/M input, $0.15/M output (gemma-4-12b-it is not listed on OpenRouter, read 2026-10-04; same reference as the Cygnet and Jev-Omni Gemma-4-12B rows) x the shim's own measured usage.prompt_tokens/completion_tokens; self-hosted on 1x H100 80GB (Lium), vLLM 0.30.0 BF16; estimated, not charged", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.02848756157635468 }, "display": "torchcast-decision-12b (Torchcast AI, Gemma-4-12B fine-tune, option-letter logprob readout)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 59.97536524790964, "I_sealed": 61.11103657326486, "base": 60.54320091058725, "excess": -3.7090589677934624, "gap": -1.1356713253552186, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 71.98506861847196, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 88.32432432432431, "hard": 67.9144385026738, "judge": 68.35860250494396, "standard": 77.39640016743407 } }, "open|noul": { "cc": 56.49758454106281, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 56.52173913043479, "judge": 50.0, "standard": 44.44444444444444 } }, "open|score": { "cc": 51.44344258419417, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 58.22709032709656, "hard": 57.48378737898936, "judge": 45.79568230097426, "standard": 44.44256954798248 } }, "sealed|choice": { "cc": 68.40340626918382, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 80.04582070800384, "hard": 60.80316742081447, "judge": 72.06703910614526, "standard": 72.28722749107035 } }, "sealed|noul": { "cc": 62.37236788127848, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 72.0, "hard": 45.63106796116505, "judge": 65.78947368421053, "standard": 85.91549295774648 } }, "sealed|score": { "cc": 52.557335569332295, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 69.05633413996496, "hard": 44.66535408470005, "judge": 55.981937871697106, "standard": 54.95489579973323 } } } }, "jevbench_score": 69.9087143535159, "key": "torchcast-decision-12b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 4, "ranked": true, "ranks": { "A": 4, "B": 5, "C": 4, "capability": 10 }, "scores": { "A": 69.9087143535159, "B": 67.81076568849952, "C": 69.9087143535159 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2243030747398734, "p50_s_raw": 0.037151537369936705, "p95_s_adjusted": 0.29825730649754406, "p95_s_raw": 0.07412865324877203 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.04707649811247, 84.79039923447887 ], "I": [ 55.00644510399965, 65.05987282834745 ], "capability": [ 67.02511371513799, 74.5041047187016 ], "composite": { "A": [ 67.22388654747238, 71.4745219700647 ], "B": [ 64.35805126511531, 70.01822593897899 ], "C": [ 56.59214410663297, 71.4745219700647 ] }, "seed": 106 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 82.93469131589123, "capability": 71.73894611323924, "intelligence": 60.54320091058725 }, "per_type": { "choice": { "accuracy": 0.7626666666666667, "calibration": 85.2114699514098, "cc": 69.64845637202428, "cc_by_type": { "choice": 69.64845637202428 }, "cc_tiered": 70.19423744382789, "cc_tiered_open": 71.98506861847196, "cc_tiered_sealed": 68.40340626918382, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 69.64845637202428 }, "noul": { "accuracy": 0.8186666666666667, "calibration": 73.02895401264989, "cc": 63.733333333333334, "cc_by_type": { "noul": 63.733333333333334 }, "cc_tiered": 59.43497621117064, "cc_tiered_open": 56.49758454106281, "cc_tiered_sealed": 62.37236788127848, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 63.733333333333334 }, "score": { "accuracy": 0.6666666666666666, "calibration": 90.56364998361398, "cc": 52.778195902361034, "cc_by_type": { "score": 52.778195902361034 }, "cc_tiered": 52.000389076763234, "cc_tiered_open": 51.44344258419417, "cc_tiered_sealed": 52.557335569332295, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 52.778195902361034 } }, "raw_on_A": { "calibration": 79.9965213222889, "capability": 69.86117451490037, "intelligence": 59.725827707511854 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 66.43077398924878, "Emphasis on Accuracy 60:20:20": 63.94347960496841, "Emphasis on Cost 20:20:60": 61.999556455891316, "Emphasis on Speed 20:60:20": 74.67237011934479, "Intelligence only": 60.54320091058725 }, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.9649595687331537, "acc_among_decisive": 0.8575418994413407, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "author submission" }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "cc-by-nc-4.0", "repo": "https://huggingface.co/torchcast-ai/torchcast-decision-12b", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (order b113eac1)", "underlying": "google/gemma-4-12B-it" }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "krishna765", "axes": { "calibration": 82.15636027953371, "cost": 49.89322575247467, "intelligence": 73.02512554551848, "speed": 86.69970536196755 }, "calibration": { "parts": { "choice": { "ece_hard": 0.10701823999728471, "ece_score": 78.59635200054306, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 78.59635200054306 }, "noul": { "abstention_rate": 0.037735849056603765, "brier": 0.059638464379526884, "ece": 0.052855966211929055, "n": 371, "score": 89.42880675761418 }, "score": { "interval90_coverage": 0.8482384823848238, "interval90_width": 0.08897922312556458, "n": 369, "nrps": 0.06373815398690165, "score": 78.44392208044394, "top_ece": 0.18369170220210976 } }, "score": 82.15636027953371 }, "capability": 77.5907429125261, "composite_ci95": { "A": [ 67.70893786128632, 70.4047103972742 ], "B": [ 67.72073941714547, 71.50656471292106 ], "C": [ 67.70893786128632, 70.4047103972742 ] }, "cost": { "basis": "ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): OpenRouter google/gemma-4-31b-it list price $0.09/M input, $0.34/M output (the exact frozen base model google/gemma-4-31B-it; same reference as Quyet-1.0-Large), read 2026-10-04 x the server's own measured usage.input_tokens; letter-mass readout, nothing generated; TorchAO FP8 dynamic on 1x GPU (Lium), deck31b 1.0.0; estimated, not charged", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.04679784 }, "display": "deck-31B (krishna765, frozen Gemma-4-31B-it, TorchAO FP8 dynamic)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 70.23424531362603, "I_sealed": 75.81600577741091, "base": 73.02512554551848, "excess": -8.155148106223118, "gap": -5.581760463784875, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 72.33184897895812, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 69.80182447310474, "judge": 64.40342781806196, "standard": 77.39640016743407 } }, "open|noul": { "cc": 77.39268461007592, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 82.6086956521739, "judge": 60.00000000000001, "standard": 88.88888888888889 } }, "open|score": { "cc": 60.978202351844075, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 73.27912483548366, "hard": 65.74514134257689, "judge": 59.223722663075364, "standard": 47.92558266171173 } }, "sealed|choice": { "cc": 83.85398241235396, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 90.68804966373514, "hard": 76.5837104072398, "judge": 87.89571694599628, "standard": 88.91489099642814 } }, "sealed|noul": { "cc": 78.05289930692999, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 96.0, "hard": 66.99029126213591, "judge": 81.57894736842107, "standard": 85.91549295774648 } }, "sealed|score": { "cc": 65.54113561294878, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 75.42107738087628, "hard": 61.37893873388316, "judge": 71.30554200754482, "standard": 60.278948895222165 } } } }, "jevbench_score": 69.33752640882658, "key": "deck31b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 5, "ranked": true, "ranks": { "A": 5, "B": 3, "C": 5, "capability": 3 }, "scores": { "A": 69.33752640882658, "B": 69.98742054141381, "C": 69.33752640882658 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.4049364184960723, "p50_s_raw": 0.12746820924803615, "p95_s_adjusted": 0.5280105819925666, "p95_s_raw": 0.18900529099628327 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.16081966059086, 84.92278583205757 ], "I": [ 68.03652261338961, 76.64977164564989 ], "capability": [ 73.77453269228936, 80.24074177369353 ], "composite": { "A": [ 67.70893786128632, 70.4047103972742 ], "B": [ 67.72073941714547, 71.50656471292106 ], "C": [ 67.70893786128632, 70.4047103972742 ] }, "seed": 34 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 82.15636027953371, "capability": 77.5907429125261, "intelligence": 73.02512554551848 }, "per_type": { "choice": { "accuracy": 0.86, "calibration": 78.59635200054306, "cc": 82.09599954529521, "cc_by_type": { "choice": 82.09599954529521 }, "cc_tiered": 78.09291569565605, "cc_tiered_open": 72.33184897895812, "cc_tiered_sealed": 83.85398241235396, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 82.09599954529521 }, "noul": { "accuracy": 0.8986666666666666, "calibration": 89.42880675761418, "cc": 79.73333333333332, "cc_by_type": { "noul": 79.73333333333332 }, "cc_tiered": 77.72279195850295, "cc_tiered_open": 77.39268461007592, "cc_tiered_sealed": 78.05289930692999, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 79.73333333333332 }, "score": { "accuracy": 0.72, "calibration": 78.44392208044394, "cc": 64.7077492239505, "cc_by_type": { "score": 64.7077492239505 }, "cc_tiered": 63.25966898239643, "cc_tiered_open": 60.978202351844075, "cc_tiered_sealed": 65.54113561294878, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 64.7077492239505 } }, "raw_on_A": { "calibration": 79.57362236322788, "capability": 77.19870727297797, "intelligence": 74.82379218272806 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 65.98518077730212, "Emphasis on Accuracy 60:20:20": 68.52137025063348, "Emphasis on Cost 20:20:60": 58.32800868887079, "Emphasis on Speed 20:60:20": 72.85260809439177, "Intelligence only": 72.71357044848186 }, "model_pin": null, "last_measured_on": "2026-10-05", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.9622641509433962, "acc_among_decisive": 0.9439775910364145, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": null }, "repo": "https://github.com/krishna-gogineni-765/deck31b" }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Eldan Ring", "axes": { "calibration": 82.95806123333315, "cost": 56.556023424877196, "intelligence": 59.53366229166759, "speed": 86.69753831262085 }, "calibration": { "parts": { "choice": { "ece_hard": 0.11234009752278087, "ece_score": 77.53198049544383, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 77.53198049544383 }, "noul": { "abstention_rate": 0.14016172506738545, "brier": 0.09737227728287662, "ece": 0.07944785238178745, "n": 371, "score": 84.11042952364252 }, "score": { "interval90_coverage": 0.9214092140921409, "interval90_width": 0.2443541102077688, "n": 369, "nrps": 0.06941649910523198, "score": 87.23177368091314, "top_ece": 0.09297401363825267 } }, "score": 82.95806123333315 }, "capability": 71.24586176250037, "composite_ci95": { "A": [ 66.45196246998344, 70.4618307059656 ], "B": [ 63.55759159512183, 69.03636277102011 ], "C": [ 53.88529001998626, 70.4618307059656 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.028062900246305422 }, "display": "Winnow-12B Q8", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 59.81914414395652, "I_sealed": 59.24818043937866, "base": 59.53366229166759, "excess": -2.0024239378603803, "gap": 0.5709637045778635, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 70.47816196296756, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 88.32432432432431, "hard": 67.9144385026738, "judge": 68.35860250494396, "standard": 69.8618668899121 } }, "open|noul": { "cc": 50.49758454106281, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 56.52173913043479, "judge": 30.000000000000004, "standard": 44.44444444444444 } }, "open|score": { "cc": 58.48168592783921, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 61.310580174513184, "hard": 65.34102213505747, "judge": 48.44185240673705, "standard": 58.40831667171895 } }, "sealed|choice": { "cc": 69.74580750216904, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 80.04582070800384, "hard": 62.33031674208144, "judge": 76.72253258845438, "standard": 68.96169478999877 } }, "sealed|noul": { "cc": 52.512988405651086, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 76.0, "hard": 28.155339805825253, "judge": 60.52631578947369, "standard": 77.46478873239437 } }, "sealed|score": { "cc": 55.48574541031586, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 72.39586027967081, "hard": 51.82211180030672, "judge": 58.61411366152696, "standard": 49.665402818839944 } } } }, "jevbench_score": 68.88542163459404, "key": "winnow-12b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 6, "ranked": true, "ranks": { "A": 6, "B": 6, "C": 6, "capability": 13 }, "scores": { "A": 68.88542163459404, "B": 66.78718784915615, "C": 67.81878724910808 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.38378054890781643, "p50_s_raw": 0.1168902744539082, "p95_s_adjusted": 0.5573951802216469, "p95_s_raw": 0.20369759011082347 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.28501959511246, 85.05351182481425 ], "I": [ 54.0544167472914, 64.20476080861759 ], "capability": [ 66.87367358483415, 74.02873882700322 ], "composite": { "A": [ 66.45196246998344, 70.4618307059656 ], "B": [ 63.55759159512183, 69.03636277102011 ], "C": [ 53.88529001998626, 70.4618307059656 ] }, "seed": 109 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 82.95806123333315, "capability": 71.24586176250037, "intelligence": 59.53366229166759 }, "per_type": { "choice": { "accuracy": 0.768, "calibration": 77.53198049544383, "cc": 70.33051353220351, "cc_by_type": { "choice": 70.33051353220351 }, "cc_tiered": 70.1119847325683, "cc_tiered_open": 70.47816196296756, "cc_tiered_sealed": 69.74580750216904, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 70.33051353220351 }, "noul": { "accuracy": 0.7786666666666666, "calibration": 84.11042952364252, "cc": 55.73333333333332, "cc_by_type": { "noul": 55.73333333333332 }, "cc_tiered": 51.505286473356946, "cc_tiered_open": 50.49758454106281, "cc_tiered_sealed": 52.512988405651086, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 55.73333333333332 }, "score": { "accuracy": 0.704, "calibration": 87.23177368091314, "cc": 56.525484762477454, "cc_by_type": { "score": 56.525484762477454 }, "cc_tiered": 56.98371566907753, "cc_tiered_open": 58.48168592783921, "cc_tiered_sealed": 55.48574541031586, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 56.525484762477454 } }, "raw_on_A": { "calibration": 80.53670554364732, "capability": 70.27835744009714, "intelligence": 60.02000933654696 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 65.19874575043826, "Emphasis on Accuracy 60:20:20": 62.8080780809742, "Emphasis on Cost 20:20:60": 61.44293550575936, "Emphasis on Speed 20:60:20": 72.37789977811173, "Intelligence only": 59.53366229166759 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0, including the applicable Gemma 4 base/derivative licence terms", "repo": "https://huggingface.co/EldanRing/Winnow-12B", "gpu": "RTX6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container", "underlying": "google/gemma-4-12B-it LoRA fine-tune, merged and exported as Q8_0 GGUF", "adapter_id": "images/winnow-12b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.8598382749326146, "acc_among_decisive": 0.9153605015673981, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "blockbrain", "axes": { "calibration": 86.96753988280044, "cost": 56.42912569481113, "intelligence": 54.83928122751086, "speed": 91.77283664982562 }, "calibration": { "parts": { "choice": { "ece_hard": 0.08631856803856107, "ece_score": 82.73628639228778, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 82.73628639228778 }, "noul": { "abstention_rate": 0.18867924528301883, "brier": 0.1059190859395021, "ece": 0.061302788041812155, "n": 371, "score": 87.73944239163757 }, "score": { "interval90_coverage": 0.978319783197832, "interval90_width": 0.46589882565492324, "n": 369, "nrps": 0.07366857377747793, "score": 90.42689086447598, "top_ece": 0.05889680446650118 } }, "score": 86.96753988280044 }, "capability": 70.90341055515565, "composite_ci95": { "A": [ 64.42215454949937, 69.93947552798573 ], "B": [ 60.49166104427309, 67.44884553134193 ], "C": [ 44.73760732604123, 67.5344744697005 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.028337561576354687 }, "display": "Cygnet (blockbrain, frozen Gemma-4-12B-it)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 58.329951877214064, "I_sealed": 51.348610577807655, "base": 54.83928122751086, "excess": 4.407953656968166, "gap": 6.9813412994064095, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 72.62577711735155, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 88.32432432432431, "hard": 67.9144385026738, "judge": 60.44825313117996, "standard": 92.46546672247803 } }, "open|noul": { "cc": 57.7536231884058, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 65.21739130434783, "judge": 50.0, "standard": 33.33333333333333 } }, "open|score": { "cc": 44.61045532588486, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 51.78845800105967, "hard": 46.33324418399161, "judge": 48.54954649888648, "standard": 31.66723951258148 } }, "sealed|choice": { "cc": 65.49755292537436, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 84.03665656640308, "hard": 56.730769230769226, "judge": 66.4804469273743, "standard": 72.28722749107035 } }, "sealed|noul": { "cc": 39.63579350399792, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 68.0, "hard": 12.621359223300965, "judge": 44.736842105263165, "standard": 71.83098591549295 } }, "sealed|score": { "cc": 48.9124853040507, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 68.44540243240824, "hard": 39.569328601993966, "judge": 55.630472156586585, "standard": 47.75535986518157 } } } }, "jevbench_score": 68.55012424151383, "key": "cygnet", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 7, "ranked": true, "ranks": { "A": 7, "B": 7, "C": 8, "capability": 14 }, "scores": { "A": 68.55012424151383, "B": 65.28560070195185, "C": 57.26499844882472 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.22363614114001393, "p50_s_raw": 0.03681807057000697, "p95_s_adjusted": 0.29728592303581536, "p95_s_raw": 0.07364296151790768 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 82.02033013748918, 88.42190039096631 ], "I": [ 49.43932563439576, 59.02934294021137 ], "capability": [ 66.3438265961016, 72.89464887942813 ], "composite": { "A": [ 64.42215454949937, 69.93947552798573 ], "B": [ 60.49166104427309, 67.44884553134193 ], "C": [ 44.73760732604123, 67.5344744697005 ] }, "seed": 23 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 86.96753988280044, "capability": 70.90341055515565, "intelligence": 54.83928122751086 }, "per_type": { "choice": { "accuracy": 0.748, "calibration": 82.73628639228778, "cc": 67.7727991815314, "cc_by_type": { "choice": 67.7727991815314 }, "cc_tiered": 69.06166502136296, "cc_tiered_open": 72.62577711735155, "cc_tiered_sealed": 65.49755292537436, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 67.7727991815314 }, "noul": { "accuracy": 0.736, "calibration": 87.73944239163757, "cc": 47.199999999999996, "cc_by_type": { "noul": 47.199999999999996 }, "cc_tiered": 48.69470834620186, "cc_tiered_open": 57.7536231884058, "cc_tiered_sealed": 39.63579350399792, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 47.199999999999996 }, "score": { "accuracy": 0.6133333333333333, "calibration": 90.42689086447598, "cc": 48.18037785294212, "cc_by_type": { "score": 48.18037785294212 }, "cc_tiered": 46.76147031496778, "cc_tiered_open": 44.61045532588486, "cc_tiered_sealed": 48.9124853040507, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 48.18037785294212 } }, "raw_on_A": { "calibration": 83.24254074674259, "capability": 69.54334852201868, "intelligence": 55.844156297294745 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 64.03016157894142, "Emphasis on Accuracy 60:20:20": 60.00734907512389, "Emphasis on Cost 20:20:60": 60.756582833038394, "Emphasis on Speed 20:60:20": 72.83760421908612, "Intelligence only": 54.83928122751087 }, "class": "system-one-open", "open": "yes", "licence": "shim MIT; weights Apache-2.0 with Google's Gemma Prohibited Use Policy", "repo": "https://github.com/blockbrain-ai/cygnet-recipe", "gpu": "RTXPRO6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container", "underlying": null, "adapter_id": "images/cygnet/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.8113207547169812, "acc_among_decisive": 0.9169435215946844, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "akhilaaa3", "axes": { "calibration": 87.04450672835743, "cost": 56.05113155939739, "intelligence": 55.530225828826175, "speed": 85.43173201875288 }, "calibration": { "parts": { "choice": { "ece_hard": 0.10691343524556984, "ece_score": 78.61731295088603, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 78.61731295088603 }, "noul": { "abstention_rate": 0.25876010781671155, "brier": 0.09426879548573859, "ece": 0.04284927173453476, "n": 371, "score": 91.43014565309304 }, "score": { "interval90_coverage": 0.94579945799458, "interval90_width": 0.32836495031616986, "n": 369, "nrps": 0.06732883113404022, "score": 91.08606158109325, "top_ece": 0.0554749686220474 } }, "score": 87.04450672835743 }, "capability": 71.2873662785918, "composite_ci95": { "A": [ 65.25170208143348, 68.96455653744843 ], "B": [ 61.61186910919615, 66.73364708003527 ], "C": [ 45.97947208044005, 67.32404729169104 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.02917173645320197 }, "display": "Jev-Omni (akhilaaa3, Gemma-4-12B merged)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 59.14991720973798, "I_sealed": 51.91053444791438, "base": 55.530225828826175, "excess": 4.6659951193853555, "gap": 7.239382761823599, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 75.02831016136747, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 73.57659641396666, "judge": 68.35860250494396, "standard": 77.39640016743407 } }, "open|noul": { "cc": 45.118702553485164, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 65.21739130434783, "judge": 10.000000000000009, "standard": 44.44444444444444 } }, "open|score": { "cc": 57.30273891436131, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 58.591582585888155, "hard": 52.24401347123031, "judge": 64.7750828511346, "standard": 55.56725205969997 } }, "sealed|choice": { "cc": 68.8767390165768, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 74.72470623013821, "hard": 62.33031674208144, "judge": 74.86033519553072, "standard": 70.07020569035596 } }, "sealed|noul": { "cc": 32.45047104291564, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 48.0, "hard": 12.621359223300965, "judge": 36.8421052631579, "standard": 57.746478873239425 } }, "sealed|score": { "cc": 54.404393284250695, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 74.52093494591816, "hard": 48.39164613508641, "judge": 58.117448525933035, "standard": 50.80203388922203 } } } }, "jevbench_score": 67.74758858028281, "key": "jev-omni", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 8, "ranked": true, "ranks": { "A": 8, "B": 8, "C": 7, "capability": 12 }, "scores": { "A": 67.74758858028281, "B": 64.89216659285307, "C": 58.02968592150493 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.45938353668898346, "p50_s_raw": 0.15469176834449172, "p95_s_adjusted": 0.6232343347743153, "p95_s_raw": 0.23661716738715766 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 82.22517864881327, 88.34500746248735 ], "I": [ 50.30228764136711, 59.27245651053757 ], "capability": [ 67.20628773376218, 73.03534010513394 ], "composite": { "A": [ 65.25170208143348, 68.96455653744843 ], "B": [ 61.61186910919615, 66.73364708003527 ], "C": [ 45.97947208044005, 67.32404729169104 ] }, "seed": 46 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 87.04450672835743, "capability": 71.2873662785918, "intelligence": 55.530225828826175 }, "per_type": { "choice": { "accuracy": 0.7693333333333333, "calibration": 78.61731295088603, "cc": 70.5010278222483, "cc_by_type": { "choice": 70.5010278222483 }, "cc_tiered": 71.95252458897214, "cc_tiered_open": 75.02831016136747, "cc_tiered_sealed": 68.8767390165768, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 70.5010278222483 }, "noul": { "accuracy": 0.688, "calibration": 91.43014565309304, "cc": 37.59999999999999, "cc_by_type": { "noul": 37.59999999999999 }, "cc_tiered": 38.7845867982004, "cc_tiered_open": 45.118702553485164, "cc_tiered_sealed": 32.45047104291564, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 37.59999999999999 }, "score": { "accuracy": 0.6906666666666667, "calibration": 91.08606158109325, "cc": 55.14278895136254, "cc_by_type": { "score": 55.14278895136254 }, "cc_tiered": 55.85356609930601, "cc_tiered_open": 57.30273891436131, "cc_tiered_sealed": 54.404393284250695, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 55.14278895136254 } }, "raw_on_A": { "calibration": 85.06452164862384, "capability": 71.04516964846138, "intelligence": 57.025817648298926 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 63.085756293696, "Emphasis on Accuracy 60:20:20": 59.82955470935818, "Emphasis on Cost 20:20:60": 60.07014592127774, "Emphasis on Speed 20:60:20": 70.45744510674369, "Intelligence only": 55.53022582882618 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0, following Gemma 4; dataset rights stated separately by the author", "repo": "https://huggingface.co/akhilaaa3/Jev-Omni", "gpu": "RTX6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container", "underlying": "google/gemma-4-12B-it fine-tuned and merged, with a trained 256-way decision head", "adapter_id": "images/jev-omni/spec.json:omni_v15.JevOmni", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7412398921832885, "acc_among_decisive": 0.9381818181818182, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": true, "api_subset": true, "author": "Levanto Labs", "axes": { "calibration": 89.8815566915413, "cost": 43.46555597245558, "intelligence": 64.29339171700846, "speed": 93.61638181239104 }, "calibration": { "parts": { "choice": { "ece_hard": 0.05752563476562504, "ece_score": 88.49487304687499, "mean_tvd": null, "n": 300, "n_ece_hard": 128, "n_tvd": 0, "score": 88.49487304687499 }, "noul": { "abstention_rate": 0.17333333333333334, "brier": 0.10062357584635416, "ece": 0.10265625000000002, "n": 150, "score": 79.46875 }, "score": { "interval90_coverage": 0.9533333333333334, "interval90_width": 0.30999999999999994, "n": 150, "nrps": 0.07244614495171442, "score": 90.51571358574762, "top_ece": 0.05861979166666666 } }, "score": 86.15977887754087 }, "capability": 77.08747420427488, "composite_ci95": { "A": [ 47.68093932220913, 51.53748391535208 ], "B": [ 46.51367777569815, 51.956941540452206 ], "C": [ 41.50299472311837, 51.53748391535208 ] }, "cost": { "basis": "Levanto list tariff, top tier: USD 0.05 per 1M input tokens, USD 10 per 1M output tokens (https://levanto.ai/pricing, read 5 Oct 2026)", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.07664458333333334 }, "display": "Sage 1.3.0 (Levanto Labs)", "endpoint_kind": "api", "full_coverage": true, "intelligence": { "I_open": 65.67742817331194, "I_sealed": 60.22459671995908, "base": 62.95101244663551, "excess": -1.0750755907082379, "gap": 5.452831453352857, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 85.05619838199219, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810813, "hard": 83.01352626612143, "judge": 84.17930125247199, "standard": 84.93093344495604 } }, "open|noul": { "cc": 54.291235334713605, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 56.52173913043479, "judge": 39.99999999999999, "standard": 55.55555555555556 } }, "open|score": { "cc": 57.68485080323005, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 71.80397727272727, "hard": 52.14120370370371, "judge": 68.19385593220339, "standard": 45.94907407407408 } }, "sealed|choice": { "cc": 79.11182203592662, "n": { "easy": 21, "hard": 60, "judge": 33, "standard": 36 }, "tiers": { "easy": 93.75, "hard": 70.04279600570615, "judge": 84.45931361968486, "standard": 81.90954773869348 } }, "sealed|noul": { "cc": 45.19912312751188, "n": { "easy": 14, "hard": 23, "judge": 21, "standard": 17 }, "tiers": { "easy": 85.71428571428572, "hard": 30.434782608695656, "judge": 61.904761904761905, "standard": 29.41176470588236 } }, "sealed|score": { "cc": 56.36284499643874, "n": { "easy": 7, "hard": 37, "judge": 17, "standard": 14 }, "tiers": { "easy": 91.625, "hard": 38.56169871794871, "judge": 57.86718749999999, "standard": 72.0775462962963 } } } }, "jevbench_score": 50.07295310117218, "key": "sage-1.3.0", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 9, "ranked": true, "ranks": { "A": 9, "B": 9, "C": 9, "capability": 5 }, "scores": { "A": 50.07295310117218, "B": 49.76845846613241, "C": 50.07295310117218 }, "speed": { "adjustment": "none (API)", "n": 124, "p50_s_adjusted": 0.15245595201849937, "p50_s_raw": 0.15245595201849937, "p95_s_adjusted": 0.2852446019649505, "p95_s_raw": 0.2852446019649505 }, "status": { "answered_ok": 600, "missing": 0, "rows": 600, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": true, "api_subset_tag": "A3", "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 81.58153931632835, 93.57066915610608 ], "I": [ 55.85968830817453, 71.57054659577504 ], "capability": [ 69.78959094534174, 81.41583283304968 ], "composite": { "A": [ 47.68093932220913, 51.53748391535208 ], "B": [ 46.51367777569815, 51.956941540452206 ], "C": [ 41.50299472311837, 51.53748391535208 ] }, "seed": 110 }, "ci95_basis": "bootstrap over the API system's items (n = A sealed items) AND over the equating offset (pool systems resampled with replacement, items resampled; B_offset = 200)", "ci95_n": 300, "complete": true, "equated": { "calibration": 89.8815566915413, "capability": 77.08747420427488, "intelligence": 64.29339171700846 }, "equating_flag": null, "equating_offset": { "C": 3.7217778140004327, "I": 1.3423792703729482 }, "g_med_used": 6.527907044061095, "lane": "api", "metric_basis": "A3 u P (sealed side = supplementary API subset A3, n=300)", "n_items": 600, "n_public": 300, "n_sealed_side": 300, "offset_ci95": { "C": [ 0.6110930594657028, 8.719922311390956 ], "I": [ -4.229711874997326, 5.822898185698653 ] }, "on_S": null, "per_type": { "choice": { "accuracy": 0.8666666666666667, "calibration": 88.49487304687499, "cc": 82.85795914480263, "cc_by_type": { "choice": 82.85795914480263 }, "cc_tiered": 82.0840102089594, "cc_tiered_open": 85.05619838199219, "cc_tiered_sealed": 79.11182203592662, "low_n": false, "n": 300, "n_by_type": { "choice": 300 }, "score": 82.85795914480263 }, "noul": { "accuracy": 0.7666666666666667, "calibration": 79.46875, "cc": 53.33333333333334, "cc_by_type": { "noul": 53.33333333333334 }, "cc_tiered": 49.74517923111274, "cc_tiered_open": 54.291235334713605, "cc_tiered_sealed": 45.19912312751188, "low_n": false, "n": 150, "n_by_type": { "noul": 150 }, "score": 53.33333333333334 }, "score": { "accuracy": 0.66, "calibration": 90.51571358574762, "cc": 56.77110861522199, "cc_by_type": { "score": 56.77110861522199 }, "cc_tiered": 57.023847899834394, "cc_tiered_open": 57.68485080323005, "cc_tiered_sealed": 56.36284499643874, "low_n": false, "n": 150, "n_by_type": { "score": 150 }, "score": 56.77110861522199 } }, "raw_on_A": { "calibration": 86.15977887754087, "capability": 74.55539566208819, "intelligence": 62.95101244663551 }, "set": "A3+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 300 }, "noul": { "invalid_rate": 0.0, "n": 150 }, "score": { "invalid_rate": 0.0, "n": 150 } }, "views": { "Balanced 33:33:33": 46.03978517152407, "Emphasis on Accuracy 60:20:20": 47.025797628375095, "Emphasis on Cost 20:20:60": 39.66698918412159, "Emphasis on Speed 20:60:20": 53.515339550995996, "Intelligence only": 48.58663328868327 }, "repo": "https://levanto.ai", "class": "decision-api", "open": "no", "licence": "proprietary hosted API", "model_pin": "levanto-sage-v1.3 (Sage 1.3.0)", "last_measured_on": "2026-10-05", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "not_ranked_because": null, "endpoint_condition": "Levanto hosted API, text only, measured from Sandy (Helsinki) on 5 Oct 2026; supplementary A3 300 sealed + P300; not evaluated on ImageJevBench.", "api_exposure_note": "A3 received by TypeSafe and Levanto without answers; retired for future draws.", "provenance": { "kind": "v1.6.0 supplementary A3 scorer output (O1S, 5 Oct 2026; strict cost over all 600 answered rows)", "source_sha256": "7ac71f7459bcb1a2062ce818c8b23dc38962793293d944abc787c06066bd2d40", "equating_pool_n": 15 }, "noul_decisive": { "supported": true, "decisive_rate": 0.8266666666666667, "acc_among_decisive": 0.9274193548387096, "valid": 150 } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "unknown", "axes": { "calibration": 87.88101858186977, "cost": 63.07359395281681, "intelligence": 42.9626698747687, "speed": 93.83356539601934 }, "calibration": { "parts": { "choice": { "ece_hard": 0.061775972135365, "ece_score": 87.644805572927, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 87.644805572927 }, "noul": { "abstention_rate": 0.29333333333333333, "brier": 0.12494153723477718, "ece": 0.06617846187638739, "n": 375, "score": 86.76430762472252 }, "score": { "interval90_coverage": 0.9573333333333334, "interval90_width": 0.49066666666666664, "n": 375, "nrps": 0.09744572815253887, "score": 89.2339425479598, "top_ece": 0.058937710444132485 } }, "score": 87.88101858186977 }, "capability": 65.42184422831923, "composite_ci95": { "A": [ 34.218232721078074, 62.094303811225494 ], "B": [ 30.257952435880583, 57.342398440511 ], "C": [ 23.762661611859777, 43.121044313351035 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01701689039408867 }, "display": "Plumb-4B (crh225, JevK5 v0.2 + LoRA)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 45.15367542782093, "I_sealed": 40.77166432171647, "base": 42.9626698747687, "excess": 1.8086234636662133, "gap": 4.382011106104457, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 63.70555942875353, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864865, "hard": 54.70273670965714, "judge": 72.31377719182596, "standard": 62.32733361239012 } }, "open|noul": { "cc": 36.11249137336094, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 39.13043478260869, "judge": 10.000000000000009, "standard": 44.44444444444444 } }, "open|score": { "cc": 35.64297548134833, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 60.38519253276966, "hard": 30.154558838586563, "judge": 43.95614933948649, "standard": 21.77893945395395 } }, "sealed|choice": { "cc": 58.16335833730948, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 65.41275589387332, "hard": 57.239819004524875, "judge": 57.16945996275605, "standard": 57.876585786426915 } }, "sealed|noul": { "cc": 23.383111546129097, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 56.00000000000001, "hard": 4.854368932038833, "judge": 36.8421052631579, "standard": 23.943661971830977 } }, "sealed|score": { "cc": 40.76852308171083, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 65.90236427761113, "hard": 40.87931496804801, "judge": 36.35703355260749, "standard": 34.597253004741354 } } } }, "jevbench_score": 48.28224126395837, "key": "plumb-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 10, "ranked": true, "ranks": { "A": 10, "B": 10, "C": 11, "capability": 22 }, "scores": { "A": 48.28224126395837, "B": 43.7169924526014, "C": 33.529334211082194 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.18999176411889493, "p50_s_raw": 0.019995882059447467, "p95_s_adjusted": 0.21772519345395266, "p95_s_raw": 0.033862596726976335 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 81.6814729990224, 88.307491492748 ], "I": [ 37.25495036908704, 47.871420386377224 ], "capability": [ 60.18380887501134, 67.22662477915313 ], "composite": { "A": [ 34.218232721078074, 62.094303811225494 ], "B": [ 30.257952435880583, 57.342398440511 ], "C": [ 23.762661611859777, 43.121044313351035 ] }, "seed": 85 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 87.88101858186977, "capability": 65.42184422831923, "intelligence": 42.9626698747687 }, "per_type": { "choice": { "accuracy": 0.6853333333333333, "calibration": 87.644805572927, "cc": 59.758627549425455, "cc_by_type": { "choice": 59.758627549425455 }, "cc_tiered": 60.93445888303151, "cc_tiered_open": 63.70555942875353, "cc_tiered_sealed": 58.16335833730948, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 59.758627549425455 }, "noul": { "accuracy": 0.6453333333333333, "calibration": 86.76430762472252, "cc": 29.066666666666663, "cc_by_type": { "noul": 29.066666666666663 }, "cc_tiered": 29.74780145974502, "cc_tiered_open": 36.11249137336094, "cc_tiered_sealed": 23.383111546129097, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 29.066666666666663 }, "score": { "accuracy": 0.576, "calibration": 89.2339425479598, "cc": 41.48168877172061, "cc_by_type": { "score": 41.48168877172061 }, "cc_tiered": 38.20574928152958, "cc_tiered_open": 35.64297548134833, "cc_tiered_sealed": 40.76852308171083, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 41.48168877172061 } }, "raw_on_A": { "calibration": 82.25783080586818, "capability": 66.15973395095179, "intelligence": 50.06163709603538 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 44.487894416524036, "Emphasis on Accuracy 60:20:20": 38.31837801553863, "Emphasis on Cost 20:20:60": 45.2973332743533, "Emphasis on Speed 20:60:20": 51.919511056412496, "Intelligence only": 31.720043803902097 }, "class": "unclassified", "open": "unknown", "licence": "not recorded", "repo": "https://huggingface.co/crh225/plumb-4b", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": null, "adapter_id": "images/plumb-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7066666666666667, "acc_among_decisive": 0.9132075471698113, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Abhishek Rai (abhishek085)", "axes": { "calibration": 64.79562625454915, "cost": 60.422063196854964, "intelligence": 43.2364603010071, "speed": 86.95618624402076 }, "calibration": { "parts": { "choice": { "ece_hard": 0.20104071779836963, "ece_score": 59.79185644032607, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 59.79185644032607 }, "noul": { "abstention_rate": 0.16000000000000003, "brier": 0.16425289707950683, "ece": 0.1444944938079116, "n": 375, "score": 71.10110123841768 }, "score": { "interval90_coverage": 0.712, "interval90_width": 0.17133333333333342, "n": 375, "nrps": 0.1542624452699586, "score": 63.493921084903704, "top_ece": 0.2879295665159836 } }, "score": 64.79562625454915 }, "capability": 54.016043277778124, "composite_ci95": { "A": [ 29.776291757874876, 55.39058631138699 ], "B": [ 26.921562908696533, 52.0349558138311 ], "C": [ 20.677980387413104, 38.465684938463184 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.020857610837438423 }, "display": "spark-s1-4b-v6 (Open Spark Jev, abhishek085)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 47.64157364296756, "I_sealed": 38.83134695904664, "base": 43.2364603010071, "excess": 6.236839041482673, "gap": 8.810226683920916, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 63.30038752190768, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 84.43243243243244, "hard": 58.47750865051904, "judge": 68.35860250494396, "standard": 54.79280033486815 } }, "open|noul": { "cc": 46.0848861283644, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 56.52173913043479, "judge": 10.000000000000009, "standard": 66.66666666666667 } }, "open|score": { "cc": 33.539447278630604, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 38.99774322763003, "hard": 44.04391027813551, "judge": 21.602744427042197, "standard": 27.706427582503657 } }, "sealed|choice": { "cc": 57.98194607005097, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 72.06414899120539, "hard": 50.113122171945705, "judge": 60.89385474860336, "standard": 62.31062938785566 } }, "sealed|noul": { "cc": 33.87772460002735, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 56.00000000000001, "hard": 2.9126213592232997, "judge": 50.0, "standard": 60.56338028169015 } }, "sealed|score": { "cc": 24.634370207061615, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 51.84151834556672, "hard": 16.688097933910274, "judge": 26.27544811817185, "standard": 24.46172381744639 } } } }, "jevbench_score": 44.90217636528639, "key": "spark-s1-4b-v6", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 11, "ranked": true, "ranks": { "A": 11, "B": 12, "C": 12, "capability": 49 }, "scores": { "A": 44.90217636528639, "B": 41.66207269525555, "C": 31.18206692033776 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.40721648099715824, "p50_s_raw": 0.1286082404985791, "p95_s_adjusted": 0.4949439602918574, "p95_s_raw": 0.17247198014592868 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 59.71050928645707, 68.28479086272019 ], "I": [ 36.48865188869242, 47.17023357174763 ], "capability": [ 48.85711252205833, 57.24976166234473 ], "composite": { "A": [ 29.776291757874876, 55.39058631138699 ], "B": [ 26.921562908696533, 52.0349558138311 ], "C": [ 20.677980387413104, 38.465684938463184 ] }, "seed": 102 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 64.79562625454915, "capability": 54.016043277778124, "intelligence": 43.2364603010071 }, "per_type": { "choice": { "accuracy": 0.6866666666666666, "calibration": 59.79185644032607, "cc": 59.92914183947025, "cc_by_type": { "choice": 59.92914183947025 }, "cc_tiered": 60.64116679597932, "cc_tiered_open": 63.30038752190768, "cc_tiered_sealed": 57.98194607005097, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 59.92914183947025 }, "noul": { "accuracy": 0.6986666666666667, "calibration": 71.10110123841768, "cc": 39.733333333333334, "cc_by_type": { "noul": 39.733333333333334 }, "cc_tiered": 39.981305364195876, "cc_tiered_open": 46.0848861283644, "cc_tiered_sealed": 33.87772460002735, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 39.733333333333334 }, "score": { "accuracy": 0.5386666666666666, "calibration": 63.493921084903704, "cc": 27.536544538601227, "cc_by_type": { "score": 27.536544538601227 }, "cc_tiered": 29.08690874284611, "cc_tiered_open": 33.539447278630604, "cc_tiered_sealed": 24.634370207061615, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 27.536544538601227 } }, "raw_on_A": { "calibration": 68.7404409420719, "capability": 57.96450731310942, "intelligence": 47.18857368414694 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 43.83191134540322, "Emphasis on Accuracy 60:20:20": 38.371611426428935, "Emphasis on Cost 20:20:60": 44.36176060417368, "Emphasis on Speed 20:60:20": 50.402171180021895, "Intelligence only": 32.33034853965725 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 (code and weights); base Qwen/Qwen3.5-4B Apache-2.0", "repo": "https://github.com/abhishek085/open-spark-jev", "gpu": "RTX6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container", "underlying": "Qwen/Qwen3.5-4B with a LoRA, read at the option-letter logits with a fitted temperature (no extra head)", "adapter_id": "images/spark-s1-4b-v6/spec.json:spark_v15.Spark", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-05", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.84, "acc_among_decisive": 0.8317460317460318, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "blockbrain", "axes": { "calibration": 88.58151013990448, "cost": 42.15074720483203, "intelligence": 56.15104294255194, "speed": 82.64878475169937 }, "calibration": { "parts": { "choice": { "ece_hard": 0.03565851721103598, "ece_score": 92.8682965577928, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 92.8682965577928 }, "noul": { "abstention_rate": 0.18867924528301883, "brier": 0.10222826353990752, "ece": 0.07688828024001365, "n": 371, "score": 84.62234395199727 }, "score": { "interval90_coverage": 0.924119241192412, "interval90_width": 0.2818428184281841, "n": 369, "nrps": 0.07387893374911293, "score": 88.25388990992337, "top_ece": 0.08052163402620996 } }, "score": 88.58151013990448 }, "capability": 72.36627654122822, "composite_ci95": { "A": [ 42.00000021718559, 44.674228699218794 ], "B": [ 40.50170443761996, 44.404221038840014 ], "C": [ 29.16666681748999, 44.674228699218794 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.08478293103448274 }, "display": "swanOne (blockbrain, Qwen3.8-Flash-Next NVFP4)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 56.23071661013901, "I_sealed": 56.07136927496486, "base": 56.15104294255194, "excess": -2.41404030726409, "gap": 0.15934733517415367, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 76.36851516458754, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 92.21621621621622, "hard": 69.80182447310474, "judge": 84.17930125247197, "standard": 69.8618668899121 } }, "open|noul": { "cc": 42.89026915113871, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 39.13043478260869, "judge": 39.99999999999999, "standard": 33.33333333333333 } }, "open|score": { "cc": 49.43336551469081, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 57.945051339203516, "hard": 45.879570270586335, "judge": 54.70148526262065, "standard": 44.382933468748654 } }, "sealed|choice": { "cc": 69.57776531270856, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 74.72470623013821, "hard": 68.43891402714931, "judge": 64.61824953445065, "standard": 76.7212710924991 } }, "sealed|noul": { "cc": 45.48583128818902, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 84.00000000000001, "hard": 18.446601941747566, "judge": 60.52631578947369, "standard": 57.746478873239425 } }, "sealed|score": { "cc": 53.150511223997015, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 70.92864532337575, "hard": 47.07089246791437, "judge": 55.79271749917136, "standard": 52.457372273711435 } } } }, "jevbench_score": 43.78640157551217, "key": "swanone", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 12, "ranked": true, "ranks": { "A": 12, "B": 11, "C": 10, "capability": 9 }, "scores": { "A": 43.78640157551217, "B": 42.950891870154436, "C": 38.348855694306984 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.7114971062168479, "p50_s_raw": 0.28074855310842395, "p95_s_adjusted": 0.7637450117617846, "p95_s_raw": 0.30687250588089227 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 83.47295009833218, 89.28636150565644 ], "I": [ 49.957046630174666, 60.997721736085786 ], "capability": [ 67.47509953288571, 74.36120880664787 ], "composite": { "A": [ 42.00000021718559, 44.674228699218794 ], "B": [ 40.50170443761996, 44.404221038840014 ], "C": [ 29.16666681748999, 44.674228699218794 ] }, "seed": 104 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 88.58151013990448, "capability": 72.36627654122822, "intelligence": 56.15104294255194 }, "per_type": { "choice": { "accuracy": 0.7786666666666666, "calibration": 92.8682965577928, "cc": 71.69462785256195, "cc_by_type": { "choice": 71.69462785256195 }, "cc_tiered": 72.97314023864806, "cc_tiered_open": 76.36851516458754, "cc_tiered_sealed": 69.57776531270856, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 71.69462785256195 }, "noul": { "accuracy": 0.744, "calibration": 84.62234395199727, "cc": 48.8, "cc_by_type": { "noul": 48.8 }, "cc_tiered": 44.188050219663864, "cc_tiered_open": 42.89026915113871, "cc_tiered_sealed": 45.48583128818902, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 48.8 }, "score": { "accuracy": 0.6506666666666666, "calibration": 88.25388990992337, "cc": 52.763382802943624, "cc_by_type": { "score": 52.763382802943624 }, "cc_tiered": 51.29193836934391, "cc_tiered_open": 49.43336551469081, "cc_tiered_sealed": 53.150511223997015, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 52.763382802943624 } }, "raw_on_A": { "calibration": 88.23312364196306, "capability": 74.40197927194893, "intelligence": 60.5708349019348 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 39.75216183669048, "Emphasis on Accuracy 60:20:20": 39.813195135198455, "Emphasis on Cost 20:20:60": 35.153484373145204, "Emphasis on Speed 20:60:20": 45.65457340538371, "Intelligence only": 39.90509729608976 }, "class": "system-one-open", "open": "yes", "licence": "patches Apache-2.0, shim MIT; weights under the Qwen licence (LICENSE-NOTICE.md)", "repo": "https://github.com/blockbrain-ai/swanone-recipe", "gpu": "RTXPRO6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container", "underlying": null, "adapter_id": "images/swanone/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.8113207547169812, "acc_among_decisive": 0.9269102990033222, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Chinh Nguyen (ncchinh)", "axes": { "calibration": 76.76007761870373, "cost": 63.43457194636937, "intelligence": 41.73350604436517, "speed": 89.46638461924755 }, "calibration": { "parts": { "choice": { "ece_hard": 0.15434999999999968, "ece_score": 69.13000000000007, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 69.13000000000007 }, "noul": { "abstention_rate": 0.24, "brier": 0.16916629888000007, "ece": 0.10974879999999991, "n": 375, "score": 78.05024000000002 }, "score": { "interval90_coverage": 0.92, "interval90_width": 0.4133333333333333, "n": 375, "nrps": 0.10737614287777777, "score": 83.09999285611111, "top_ece": 0.11531199999999998 } }, "score": 76.76007761870373 }, "capability": 59.24679183153445, "composite_ci95": { "A": [ 30.84848869900701, 57.23347326869325 ], "B": [ 27.312995844624446, 52.90407172237297 ], "C": [ 21.422561596532642, 39.74546754770365 ] }, "cost": { "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): DeepInfra Qwen/Qwen3.5-4B list price $0.03/M input (the exact base weights; not on OpenRouter; same reference as reflex-4b, jobe and typecastlm), read 2026-09-24 x the package's own measured usage.input_tokens; one forward pass, nothing generated; run in process on 1x H100 80GB (Lium), quyet 1.0.0; estimated, not charged", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01655189039408867 }, "display": "Quyet-1.0-Medium (Chinh Nguyen, Qwen3.5-4B decoder, option-letter logits)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 42.75300136168062, "I_sealed": 40.71401072704972, "base": 41.73350604436517, "excess": -0.5343970078073448, "gap": 2.038990634630899, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 70.4971468014646, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864865, "hard": 67.9144385026738, "judge": 72.31377719182596, "standard": 69.8618668899121 } }, "open|noul": { "cc": 16.34230503795721, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": 4.347826086956519, "judge": 0.0, "standard": 44.44444444444444 } }, "open|score": { "cc": 41.41955224562005, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 47.18969696969697, "hard": 35.19814814814814, "judge": 54.641694915254234, "standard": 31.14407407407409 } }, "sealed|choice": { "cc": 59.00724128213771, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 65.41275589387332, "hard": 55.20361990950226, "judge": 55.30726256983241, "standard": 68.96169478999877 } }, "sealed|noul": { "cc": 25.985534052552406, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 60.00000000000001, "hard": 12.621359223300965, "judge": 26.315789473684205, "standard": 35.2112676056338 } }, "sealed|score": { "cc": 37.14925684645904, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 55.4977304964539, "hard": 32.33151960784312, "judge": 40.893173652694614, "standard": 31.994619289340076 } } } }, "jevbench_score": 43.58925238992216, "key": "quyet-1-0-medium", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 13, "ranked": true, "ranks": { "A": 13, "B": 14, "C": 13, "capability": 37 }, "scores": { "A": 43.58925238992216, "B": 39.63222739545648, "C": 30.270314159668164 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3048233319917927, "p50_s_raw": 0.07741166599589633, "p95_s_adjusted": 0.3709482546968502, "p95_s_raw": 0.1104741273484251 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 71.2671834460676, 80.2774031670074 ], "I": [ 36.06269585061453, 46.7313636734712 ], "capability": [ 54.568687859056396, 62.84407796698955 ], "composite": { "A": [ 30.84848869900701, 57.23347326869325 ], "B": [ 27.312995844624446, 52.90407172237297 ], "C": [ 21.422561596532642, 39.74546754770365 ] }, "seed": 87 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 76.76007761870373, "capability": 59.24679183153445, "intelligence": 41.73350604436517 }, "per_type": { "choice": { "accuracy": 0.7013333333333334, "calibration": 69.13000000000007, "cc": 61.804799029963135, "cc_by_type": { "choice": 61.804799029963135 }, "cc_tiered": 64.75219404180115, "cc_tiered_open": 70.4971468014646, "cc_tiered_sealed": 59.00724128213771, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 61.804799029963135 }, "noul": { "accuracy": 0.64, "calibration": 78.05024000000002, "cc": 28.000000000000004, "cc_by_type": { "noul": 28.000000000000004 }, "cc_tiered": 21.163919545254807, "cc_tiered_open": 16.34230503795721, "cc_tiered_sealed": 25.985534052552406, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 28.000000000000004 }, "score": { "accuracy": 0.56, "calibration": 83.09999285611111, "cc": 38.22100877192973, "cc_by_type": { "score": 38.22100877192973 }, "cc_tiered": 39.28440454603954, "cc_tiered_open": 41.41955224562005, "cc_tiered_sealed": 37.14925684645904, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 38.22100877192973 } }, "raw_on_A": { "calibration": 77.01676225205841, "capability": 61.22371994730187, "intelligence": 45.430677642545334 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 41.05875168344406, "Emphasis on Accuracy 60:20:20": 35.24739388678317, "Emphasis on Cost 20:20:60": 42.25762883934701, "Emphasis on Speed 20:60:20": 47.54937088611952, "Intelligence only": 29.074657383286326 }, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.76, "acc_among_decisive": 0.8421052631578947, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "author submission" }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Medium", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (order 3643732b)", "underlying": "Qwen/Qwen3.5-4B" }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Interfaze", "axes": { "calibration": 82.32371666823741, "cost": 61.79891447724963, "intelligence": 41.084702071234844, "speed": 88.00251499520694 }, "calibration": { "parts": { "choice": { "ece_hard": 0.09976787636606844, "ece_score": 80.04642472678631, "mean_tvd": null, "n": 735, "n_ece_hard": 305, "n_tvd": 0, "score": 80.04642472678631 }, "noul": { "abstention_rate": 0.15633423180592987, "brier": 0.15547862054178035, "ece": 0.09804350768859321, "n": 371, "score": 80.39129846228136 }, "score": { "interval90_coverage": 0.9728997289972899, "interval90_width": 0.7095754290876237, "n": 369, "nrps": 0.12033260727134382, "score": 86.53342681564456, "top_ece": 0.07449942820788243 } }, "score": 82.32371666823741 }, "capability": 61.70420936973613, "composite_ci95": { "A": [ 28.718912479130207, 50.73774516604421 ], "B": [ 25.322064520088336, 46.59426967064348 ], "C": [ 19.943689221618207, 35.23454525419738 ] }, "cost": { "basis": "latest published v1.5 row (v1.5.8 combined file) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3); self-hosted estimate, not measured hosted billing", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01876590517241379 }, "display": "lev (Interfaze AI, Qwen3.5-4B + LoRA r32, label-token readout + candidate-path head, per-bucket temperatures)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 45.778585099687206, "I_sealed": 36.39081904278248, "base": 41.084702071234844, "excess": 6.81437841446648, "gap": 9.387766056904724, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 61.009098246344166, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864865, "hard": 50.92796476879523, "judge": 68.35860250494396, "standard": 62.32733361239012 } }, "open|noul": { "cc": 56.946169772256724, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": 73.91304347826086, "judge": 50.0, "standard": 33.33333333333333 } }, "open|score": { "cc": 19.38048728046074, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 22.564349870758527, "hard": 19.589685248514364, "judge": 26.558897550056017, "standard": 6.602544644811692 } }, "sealed|choice": { "cc": 54.52900312820748, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 78.71554208853745, "hard": 48.585972850678715, "judge": 54.37616387337058, "standard": 54.55105308535534 } }, "sealed|noul": { "cc": 29.603595615594433, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 63.99999999999999, "hard": 2.9126213592232997, "judge": 36.8421052631579, "standard": 54.92957746478873 } }, "sealed|score": { "cc": 25.039858384545557, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 47.25762460028372, "hard": 16.968805028678304, "judge": 35.272820946831395, "standard": 14.723638144982221 } } } }, "jevbench_score": 42.177760514319104, "key": "lev", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 14, "ranked": true, "ranks": { "A": 14, "B": 15, "C": 14, "capability": 33 }, "scores": { "A": 42.177760514319104, "B": 38.201113541708786, "C": 29.290111468277157 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3546039809356444, "p50_s_raw": 0.10230199046782218, "p95_s_adjusted": 0.446688625391107, "p95_s_raw": 0.1483443126955535 }, "status": { "answered_ok": 1475, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 76.63846170548479, 84.6851724512148 ], "I": [ 35.04763127374306, 44.44652580243387 ], "capability": [ 56.747890795417284, 63.66051033763406 ], "composite": { "A": [ 28.718912479130207, 50.73774516604421 ], "B": [ 25.322064520088336, 46.59426967064348 ], "C": [ 19.943689221618207, 35.23454525419738 ] }, "seed": 59 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 82.32371666823741, "capability": 61.70420936973613, "intelligence": 41.084702071234844 }, "per_type": { "choice": { "accuracy": 0.664, "calibration": 80.04642472678631, "cc": 57.030398908708534, "cc_by_type": { "choice": 57.030398908708534 }, "cc_tiered": 57.76905068727582, "cc_tiered_open": 61.009098246344166, "cc_tiered_sealed": 54.52900312820748, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 57.030398908708534 }, "noul": { "accuracy": 0.6906666666666667, "calibration": 80.39129846228136, "cc": 38.13333333333333, "cc_by_type": { "noul": 38.13333333333333 }, "cc_tiered": 43.274882693925576, "cc_tiered_open": 56.946169772256724, "cc_tiered_sealed": 29.603595615594433, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 38.13333333333333 }, "score": { "accuracy": 0.4533333333333333, "calibration": 86.53342681564456, "cc": 23.66953381268505, "cc_by_type": { "score": 23.66953381268505 }, "cc_tiered": 22.210172832503147, "cc_tiered_open": 19.38048728046074, "cc_tiered_sealed": 25.039858384545557, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 23.66953381268505 } }, "raw_on_A": { "calibration": 79.50973335881918, "capability": 61.67507354014826, "intelligence": 43.84041372147735 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.020000000000000018, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 39.03925349464146, "Emphasis on Accuracy 60:20:20": 33.569486237063714, "Emphasis on Cost 20:20:60": 40.07113476989396, "Emphasis on Speed 20:60:20": 45.246505158344306, "Intelligence only": 27.739614243661805 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/interfaze-ai/lev", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100 80GB), offline (HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE), credential-free, server bound to 127.0.0.1, harness on the same pod over loopback HTTP; Lium pod 08a1ff4a-248a-4ff2-948e-ea558f99e808 (huid cosmic-hawk-4f, node calm-fox-65), 1x NVIDIA H100 80GB HBM3, United States, USD 1.30/h", "underlying": "Qwen/Qwen3.5-4B", "adapter_id": "/home/flori/jobs/jevbench-add-requests-20260919/r39/outputs/meta/specs/lev.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": "interfaze-ai/lev@7bdc748dffebd85b57ee0dbea8f994c6354fed31 + Abhinavexists/lev@cf104b69329302e4eac674a730c71f3511047db8", "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.8436657681940701, "acc_among_decisive": 0.8274760383386581, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Igor L. / NInfer contributors", "axes": { "calibration": 90.37232453477458, "cost": 42.53362878447125, "intelligence": 48.980301026876646, "speed": 89.33173525631216 }, "calibration": { "parts": { "choice": { "ece_hard": 0.03727735037984102, "ece_score": 92.54452992403179, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 92.54452992403179 }, "noul": { "abstention_rate": 0.24528301886792447, "brier": 0.10381961467316755, "ece": 0.05486291160058669, "n": 371, "score": 89.02741767988266 }, "score": { "interval90_coverage": 0.94579945799458, "interval90_width": 0.40876242095754284, "n": 369, "nrps": 0.08645101367263491, "score": 89.5450260004093, "top_ece": 0.06132423315958956 } }, "score": 90.37232453477458 }, "capability": 69.67631278082561, "composite_ci95": { "A": [ 30.27504047421702, 44.819854782587115 ], "B": [ 28.319423416872812, 43.496842238713214 ], "C": [ 21.02433366265071, 36.222113908221345 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.08232764778325123 }, "display": "NInfer Qwen3.8-Flash-Next mixed", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 48.827164297755125, "I_sealed": 49.13343775599816, "base": 48.980301026876646, "excess": -2.8796611006812776, "gap": -0.30627345824303376, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 70.22063841159643, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 92.21621621621622, "hard": 60.36489462094999, "judge": 76.26895187870797, "standard": 69.8618668899121 } }, "open|noul": { "cc": 36.42788129744652, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 30.434782608695656, "judge": 19.999999999999996, "standard": 55.55555555555556 } }, "open|score": { "cc": 39.83297318422244, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 63.934842929432264, "hard": 29.854097365310107, "judge": 50.77541876771969, "standard": 31.326121574196307 } }, "sealed|choice": { "cc": 68.53578282889805, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 77.38526346907102, "hard": 61.82126696832577, "judge": 68.34264432029796, "standard": 77.82978199285628 } }, "sealed|noul": { "cc": 34.31380022598546, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 68.0, "hard": -0.9708737864077666, "judge": 52.63157894736843, "standard": 60.56338028169015 } }, "sealed|score": { "cc": 44.55073021311098, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 66.34071381045655, "hard": 36.06656765199164, "judge": 48.53812828291905, "standard": 44.64296643196477 } } } }, "jevbench_score": 41.96785076957165, "key": "ninfer-qwen3.8-flash-next", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 15, "ranked": true, "ranks": { "A": 15, "B": 13, "C": 15, "capability": 15 }, "scores": { "A": 41.96785076957165, "B": 40.09260225022487, "C": 29.144340812202532 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3100576108321548, "p50_s_raw": 0.08002880541607738, "p95_s_adjusted": 0.37616993160918355, "p95_s_raw": 0.11308496580459178 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 84.49584722295648, 90.3923314283596 ], "I": [ 42.680473865516596, 53.996834913811625 ], "capability": [ 64.4871361894938, 71.33725830271337 ], "composite": { "A": [ 30.27504047421702, 44.819854782587115 ], "B": [ 28.319423416872812, 43.496842238713214 ], "C": [ 21.02433366265071, 36.222113908221345 ] }, "seed": 73 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 90.37232453477458, "capability": 69.67631278082561, "intelligence": 48.980301026876646 }, "per_type": { "choice": { "accuracy": 0.7613333333333333, "calibration": 92.54452992403179, "cc": 69.47794208197946, "cc_by_type": { "choice": 69.47794208197946 }, "cc_tiered": 69.37821062024724, "cc_tiered_open": 70.22063841159643, "cc_tiered_sealed": 68.53578282889805, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 69.47794208197946 }, "noul": { "accuracy": 0.696, "calibration": 89.02741767988266, "cc": 39.19999999999999, "cc_by_type": { "noul": 39.19999999999999 }, "cc_tiered": 35.37084076171599, "cc_tiered_open": 36.42788129744652, "cc_tiered_sealed": 34.31380022598546, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 39.19999999999999 }, "score": { "accuracy": 0.5653333333333334, "calibration": 89.5450260004093, "cc": 43.98717489610087, "cc_by_type": { "score": 43.98717489610087 }, "cc_tiered": 42.19185169866671, "cc_tiered_open": 39.83297318422244, "cc_tiered_sealed": 44.55073021311098, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 43.98717489610087 } }, "raw_on_A": { "calibration": 86.91372207226564, "capability": 70.43814133335954, "intelligence": 53.962560594453436 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 37.79450787350779, "Emphasis on Accuracy 60:20:20": 36.18545094340192, "Emphasis on Cost 20:20:60": 33.99296424621917, "Emphasis on Speed 20:60:20": 44.79615638245711, "Intelligence only": 34.01333411780027 }, "class": "native-logit", "open": "yes", "licence": "Qwen Community License 1.0 (model); Apache-2.0 (engine)", "repo": "https://github.com/igorls/ninfer", "gpu": "RTXPRO6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container", "underlying": "Qwen/Qwen3.8-Flash-Next, mixed NVFP4/FP8/INT4 NInfer artifact", "adapter_id": "images/ninfer-qwen3.8-flash-next/spec.json:ninfer_native.NInferNativeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7547169811320755, "acc_among_decisive": 0.9321428571428572, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "us (GitHub)", "axes": { "calibration": 73.06634984647151, "cost": 58.85554343293177, "intelligence": 41.85041350559534, "speed": 75.55955847735143 }, "calibration": { "parts": { "choice": { "ece_hard": 0.16559413680781765, "ece_score": 66.88117263843647, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 66.88117263843647 }, "noul": { "abstention_rate": 0.14555256064690025, "brier": 0.16930220654986547, "ece": 0.14746549865229122, "n": 371, "score": 70.50690026954176 }, "score": { "interval90_coverage": 0.8644986449864499, "interval90_width": 0.325880758807588, "n": 369, "nrps": 0.10296637523035226, "score": 81.81097663143632, "top_ece": 0.1304070460704606 } }, "score": 73.06634984647151 }, "capability": 57.458381676033426, "composite_ci95": { "A": [ 29.136540211681222, 52.90884989606441 ], "B": [ 26.3171992505717, 49.89250384145016 ], "C": [ 20.233708480334183, 36.74225687226694 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.023522413793103446 }, "display": "jev-local (Qwen3.5-9B)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 43.39498650106927, "I_sealed": 40.30584051012142, "base": 41.85041350559534, "excess": 0.5157583485096047, "gap": 3.0891459909478485, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 56.18661240978703, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 53.2972972972973, "hard": 49.040578798364265, "judge": 52.537903757415954, "standard": 77.39640016743407 } }, "open|noul": { "cc": 34.189786059351285, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 30.434782608695656, "judge": 30.000000000000004, "standard": 22.222222222222232 } }, "open|score": { "cc": 39.8085610340695, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 40.41969696969697, "hard": 51.80083333333333, "judge": 22.97271186440677, "standard": 40.77222222222221 } }, "sealed|choice": { "cc": 47.63585353447873, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 58.7613627965413, "hard": 39.42307692307691, "judge": 48.78957169459964, "standard": 56.76807488606972 } }, "sealed|noul": { "cc": 35.679356877082625, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 68.0, "hard": 14.563106796116498, "judge": 42.10526315789473, "standard": 52.112676056338024 } }, "sealed|score": { "cc": 37.602311118802916, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 64.23531914893618, "hard": 29.158455882352953, "judge": 39.61335329341317, "standard": 38.15695431472081 } } } }, "jevbench_score": 41.328564334669615, "key": "jev-local", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 16, "ranked": true, "ranks": { "A": 16, "B": 16, "C": 16, "capability": 41 }, "scores": { "A": 41.328564334669615, "B": 38.19938992160457, "C": 28.700391899076113 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.8806604199111462, "p50_s_raw": 0.3653302099555731, "p95_s_adjusted": 3.156717182928696, "p95_s_raw": 1.5033585914643481 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 67.57161963107492, 76.81314888025398 ], "I": [ 36.13732889468463, 46.54627874317067 ], "capability": [ 53.19292305173127, 60.44357521274037 ], "composite": { "A": [ 29.136540211681222, 52.90884989606441 ], "B": [ 26.3171992505717, 49.89250384145016 ], "C": [ 20.233708480334183, 36.74225687226694 ] }, "seed": 45 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 73.06634984647151, "capability": 57.458381676033426, "intelligence": 41.85041350559534 }, "per_type": { "choice": { "accuracy": 0.6026666666666667, "calibration": 66.88117263843647, "cc": 49.186741566647385, "cc_by_type": { "choice": 49.186741566647385 }, "cc_tiered": 51.91123297213288, "cc_tiered_open": 56.18661240978703, "cc_tiered_sealed": 47.63585353447873, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 49.186741566647385 }, "noul": { "accuracy": 0.696, "calibration": 70.50690026954176, "cc": 39.19999999999999, "cc_by_type": { "noul": 39.19999999999999 }, "cc_tiered": 34.93457146821696, "cc_tiered_open": 34.189786059351285, "cc_tiered_sealed": 35.679356877082625, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 39.19999999999999 }, "score": { "accuracy": 0.5653333333333334, "calibration": 81.81097663143632, "cc": 39.00943859649125, "cc_by_type": { "score": 39.00943859649125 }, "cc_tiered": 38.70543607643621, "cc_tiered_open": 39.8085610340695, "cc_tiered_sealed": 37.602311118802916, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 39.00943859649125 } }, "raw_on_A": { "calibration": 73.6085835138451, "capability": 59.88003789504907, "intelligence": 46.151492276253045 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 38.834988968034565, "Emphasis on Accuracy 60:20:20": 34.37288631458028, "Emphasis on Cost 20:20:60": 39.75999364263786, "Emphasis on Speed 20:60:20": 43.46632094435822, "Intelligence only": 29.31968172619126 }, "class": "jev-rebuild", "open": "yes", "licence": "no licence stated in the repository (public code); Apache-2.0 base weights", "repo": "https://github.com/us/jev-local", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "Qwen/Qwen3.5-9B, frozen, per-option mean log-probability", "adapter_id": "images/jev-local/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.8544474393530997, "acc_among_decisive": 0.8233438485804416, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Mapika", "axes": { "calibration": 89.63735637833331, "cost": 64.54376873042901, "intelligence": 40.12597268917964, "speed": 91.78549329698745 }, "calibration": { "parts": { "choice": { "ece_hard": 0.04618531250000016, "ece_score": 90.76293749999998, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 90.76293749999998 }, "noul": { "abstention_rate": 0.3573333333333333, "brier": 0.1263556454400002, "ece": 0.05272693333333338, "n": 375, "score": 89.45461333333333 }, "score": { "interval90_coverage": 0.9733333333333334, "interval90_width": 0.5584444444444441, "n": 375, "nrps": 0.09968243396666672, "score": 88.69451830166665, "top_ece": 0.06321360000000002 } }, "score": 89.63735637833331 }, "capability": 64.88166453375648, "composite_ci95": { "A": [ 28.727001690110686, 49.92424966763833 ], "B": [ 25.10971569724804, 45.24030143351226 ], "C": [ 19.949306729243542, 34.66961782474885 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.015201077586206898 }, "display": "decider-4b v2 (Mapika)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 45.24914817619414, "I_sealed": 35.00279720216514, "base": 40.12597268917964, "excess": 7.672963331590756, "gap": 10.246350974029, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 73.21703197407996, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 84.43243243243244, "hard": 69.80182447310474, "judge": 76.26895187870797, "standard": 69.8618668899121 } }, "open|noul": { "cc": 26.461697722567287, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 39.13043478260869, "judge": -9.999999999999998, "standard": 33.33333333333333 } }, "open|score": { "cc": 36.06871483193517, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 48.519696969696966, "hard": 36.743888888888875, "judge": 38.64050847457627, "standard": 24.635185185185193 } }, "sealed|choice": { "cc": 56.802191819905744, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 69.40359175227256, "hard": 50.62217194570135, "judge": 57.16945996275605, "standard": 62.31062938785566 } }, "sealed|noul": { "cc": 11.645451862940547, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 48.0, "hard": -12.621359223300976, "judge": 23.684210526315795, "standard": 23.943661971830977 } }, "sealed|score": { "cc": 36.56074792364913, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 54.016382978723385, "hard": 31.53129901960784, "judge": 40.6503592814371, "standard": 31.75741116751262 } } } }, "jevbench_score": 41.240894350857424, "key": "decider-4b-v2", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 17, "ranked": true, "ranks": { "A": 17, "B": 17, "C": 17, "capability": 23 }, "scores": { "A": 41.240894350857424, "B": 36.84955227461344, "C": 28.639509965873213 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.19992743488401174, "p50_s_raw": 0.024963717442005873, "p95_s_adjusted": 0.3315723263658583, "p95_s_raw": 0.09078616318292916 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 84.18439184191017, 90.03628745741203 ], "I": [ 34.68576750629042, 43.56524556183246 ], "capability": [ 60.38851961863991, 66.00729141285105 ], "composite": { "A": [ 28.727001690110686, 49.92424966763833 ], "B": [ 25.10971569724804, 45.24030143351226 ], "C": [ 19.949306729243542, 34.66961782474885 ] }, "seed": 28 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.63735637833331, "capability": 64.88166453375648, "intelligence": 40.12597268917964 }, "per_type": { "choice": { "accuracy": 0.6933333333333334, "calibration": 90.76293749999998, "cc": 60.781713289694295, "cc_by_type": { "choice": 60.781713289694295 }, "cc_tiered": 65.00961189699285, "cc_tiered_open": 73.21703197407996, "cc_tiered_sealed": 56.802191819905744, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 60.781713289694295 }, "noul": { "accuracy": 0.592, "calibration": 89.45461333333333, "cc": 18.399999999999995, "cc_by_type": { "noul": 18.399999999999995 }, "cc_tiered": 19.053574792753917, "cc_tiered_open": 26.461697722567287, "cc_tiered_sealed": 11.645451862940547, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 18.399999999999995 }, "score": { "accuracy": 0.512, "calibration": 88.69451830166665, "cc": 36.877482456140385, "cc_by_type": { "score": 36.877482456140385 }, "cc_tiered": 36.31473137779215, "cc_tiered_open": 36.06871483193517, "cc_tiered_sealed": 36.56074792364913, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 36.877482456140385 } }, "raw_on_A": { "calibration": 88.78997857788461, "capability": 66.11763148027418, "intelligence": 43.44528438266377 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 37.65578397809399, "Emphasis on Accuracy 60:20:20": 31.83485819738892, "Emphasis on Cost 20:20:60": 39.12905962819718, "Emphasis on Speed 20:60:20": 44.051943214406094, "Intelligence only": 25.842630080539234 }, "class": "system-one-open", "open": "yes", "licence": "Apache-2.0 (package and weights)", "repo": "https://github.com/Mapika/decider", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": null, "adapter_id": "images/decider-4b-v2/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6426666666666667, "acc_among_decisive": 0.921161825726141, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "unknown", "axes": { "calibration": 89.88230801873149, "cost": 63.07359395281681, "intelligence": 38.53201254374527, "speed": 93.9472917853559 }, "calibration": { "parts": { "choice": { "ece_hard": 0.05458773337668626, "ece_score": 89.08245332466275, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 89.08245332466275 }, "noul": { "abstention_rate": 0.33692722371967654, "brier": 0.11477714806826486, "ece": 0.053786553580231086, "n": 371, "score": 89.24268928395378 }, "score": { "interval90_coverage": 0.9566395663956639, "interval90_width": 0.4780939476061429, "n": 369, "nrps": 0.09977232020092458, "score": 91.32178144757796, "top_ece": 0.03689602542375804 } }, "score": 89.88230801873149 }, "capability": 64.20716028123837, "composite_ci95": { "A": [ 24.2437041451056, 49.59495089694384 ], "B": [ 20.90887497172086, 45.03359602914599 ], "C": [ 16.835905656323334, 34.440938122877675 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01701689039408867 }, "display": "JevK5 v0.3 (4B)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 38.333624208131226, "I_sealed": 38.730400879359316, "base": 38.53201254374527, "excess": -2.970164313666333, "gap": -0.39677667122808913, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 60.58350447013268, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864865, "hard": 60.36489462094999, "judge": 64.40342781806196, "standard": 47.25826705734617 } }, "open|noul": { "cc": 20.41614906832298, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 42.85714285714286, "hard": 47.82608695652173, "judge": -9.999999999999998, "standard": 0.0 } }, "open|score": { "cc": 34.00121908593804, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 53.18739024047374, "hard": 34.10450662572488, "judge": 37.96632221813109, "standard": 18.253903730806933 } }, "sealed|choice": { "cc": 58.82061791071595, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 68.07331313280615, "hard": 55.712669683257914, "judge": 59.031657355679705, "standard": 60.09360758714129 } }, "sealed|noul": { "cc": 22.94594773546748, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 48.0, "hard": 0.9708737864077666, "judge": 39.47368421052631, "standard": 29.5774647887324 } }, "sealed|score": { "cc": 34.42463699189452, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 65.98468618250719, "hard": 27.333586499919292, "judge": 34.53681846177148, "standard": 32.6584411757232 } } } }, "jevbench_score": 37.36493548751909, "key": "jevk5-v0.3-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 18, "ranked": true, "ranks": { "A": 18, "B": 21, "C": 18, "capability": 24 }, "scores": { "A": 37.36493548751909, "B": 33.16714732620591, "C": 25.947871866332704 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.18763320203870534, "p50_s_raw": 0.018816601019352674, "p95_s_adjusted": 0.21476382557302715, "p95_s_raw": 0.03238191278651357 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 83.86370931431408, 89.71025718681356 ], "I": [ 32.336664057232596, 43.498424875066746 ], "capability": [ 59.05664331423508, 65.72933144554092 ], "composite": { "A": [ 24.2437041451056, 49.59495089694384 ], "B": [ 20.90887497172086, 45.03359602914599 ], "C": [ 16.835905656323334, 34.440938122877675 ] }, "seed": 48 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.88230801873149, "capability": 64.20716028123837, "intelligence": 38.53201254374527 }, "per_type": { "choice": { "accuracy": 0.688, "calibration": 89.08245332466275, "cc": 60.099656129515054, "cc_by_type": { "choice": 60.099656129515054 }, "cc_tiered": 59.70206119042432, "cc_tiered_open": 60.58350447013268, "cc_tiered_sealed": 58.82061791071595, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 60.099656129515054 }, "noul": { "accuracy": 0.6213333333333333, "calibration": 89.24268928395378, "cc": 24.26666666666666, "cc_by_type": { "noul": 24.26666666666666 }, "cc_tiered": 21.681048401895232, "cc_tiered_open": 20.41614906832298, "cc_tiered_sealed": 22.94594773546748, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 24.26666666666666 }, "score": { "accuracy": 0.552, "calibration": 91.32178144757796, "cc": 35.814600564516994, "cc_by_type": { "score": 35.814600564516994 }, "cc_tiered": 34.21292803891628, "cc_tiered_open": 34.00121908593804, "cc_tiered_sealed": 34.42463699189452, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 35.814600564516994 } }, "raw_on_A": { "calibration": 87.1680130057895, "capability": 65.06324188254719, "intelligence": 42.95847075930487 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 33.96793409727579, "Emphasis on Accuracy 60:20:20": 28.454799805414766, "Emphasis on Cost 20:20:60": 35.283091257812984, "Emphasis on Speed 20:60:20": 40.26908605684886, "Intelligence only": 22.883638070578947 }, "class": "unclassified", "open": "unknown", "licence": "not recorded", "repo": "https://huggingface.co/alibiserikbay/JevK5", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": null, "adapter_id": "images/jevk5-v0.3-4b/spec.json:jevk5_v15.JevK5Warm", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6630727762803235, "acc_among_decisive": 0.9471544715447154, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Wayfind (metask-ai)", "axes": { "calibration": 84.41575352936586, "cost": 57.7311327584882, "intelligence": 39.166454612935595, "speed": 90.78239008886905 }, "calibration": { "parts": { "choice": { "ece_hard": 0.06577096449867435, "ece_score": 86.84580710026512, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 86.84580710026512 }, "noul": { "abstention_rate": 0.2506738544474394, "brier": 0.14719486396931378, "ece": 0.09558018432686369, "n": 371, "score": 80.88396313462727 }, "score": { "interval90_coverage": 0.9728997289972899, "interval90_width": 0.6723125564588976, "n": 369, "nrps": 0.11633460342816504, "score": 85.51749035320519, "top_ece": 0.08665779475386558 } }, "score": 84.41575352936586 }, "capability": 61.79110407115073, "composite_ci95": { "A": [ 24.35782606760525, 47.92198520846029 ], "B": [ 21.36935197657226, 43.913080458446345 ], "C": [ 16.915156991392532, 33.27915639476408 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.025642610837438424 }, "display": "metask-jev-4b", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 41.89224435585361, "I_sealed": 36.440664870017585, "base": 39.166454612935595, "excess": 2.878191843397784, "gap": 5.451579485836028, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 60.50866528870256, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675674, "hard": 52.815350739226176, "judge": 60.44825313117996, "standard": 69.8618668899121 } }, "open|noul": { "cc": 37.846790890269155, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 56.52173913043479, "judge": 0.0, "standard": 33.33333333333333 } }, "open|score": { "cc": 27.321276888589132, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 36.68821311926259, "hard": 13.269014128057655, "judge": 46.73482367537818, "standard": 21.622014114131783 } }, "sealed|choice": { "cc": 56.24961082249847, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 73.3944276106718, "hard": 48.076923076923066, "judge": 61.824953445065184, "standard": 55.65956398571254 } }, "sealed|noul": { "cc": 22.8393257860911, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 63.99999999999999, "hard": 2.9126213592232997, "judge": 23.684210526315795, "standard": 40.8450704225352 } }, "sealed|score": { "cc": 30.23305800146319, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 50.21133272720072, "hard": 17.595550281640328, "judge": 37.810328121508064, "standard": 34.153030898172844 } } } }, "jevbench_score": 37.34931073616377, "key": "metask-jev-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 19, "ranked": true, "ranks": { "A": 19, "B": 19, "C": 19, "capability": 32 }, "scores": { "A": 37.34931073616377, "B": 33.62317034896201, "C": 25.937021344558172 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.25794456995790827, "p50_s_raw": 0.05397228497895412, "p95_s_adjusted": 0.3237685059779324, "p95_s_raw": 0.08688425298896618 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.04800226228174, 85.90669927321129 ], "I": [ 32.96914634416673, 43.538341958330456 ], "capability": [ 56.20112405588935, 63.92462857658634 ], "composite": { "A": [ 24.35782606760525, 47.92198520846029 ], "B": [ 21.36935197657226, 43.913080458446345 ], "C": [ 16.915156991392532, 33.27915639476408 ] }, "seed": 66 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 84.41575352936586, "capability": 61.79110407115073, "intelligence": 39.166454612935595 }, "per_type": { "choice": { "accuracy": 0.668, "calibration": 86.84580710026512, "cc": 57.541941778842954, "cc_by_type": { "choice": 57.541941778842954 }, "cc_tiered": 58.379138055600514, "cc_tiered_open": 60.50866528870256, "cc_tiered_sealed": 56.24961082249847, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 57.541941778842954 }, "noul": { "accuracy": 0.6506666666666666, "calibration": 80.88396313462727, "cc": 30.133333333333322, "cc_by_type": { "noul": 30.133333333333322 }, "cc_tiered": 30.34305833818013, "cc_tiered_open": 37.846790890269155, "cc_tiered_sealed": 22.8393257860911, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 30.133333333333322 }, "score": { "accuracy": 0.48, "calibration": 85.51749035320519, "cc": 29.198392850325618, "cc_by_type": { "score": 29.198392850325618 }, "cc_tiered": 28.77716744502616, "cc_tiered_open": 27.321276888589132, "cc_tiered_sealed": 30.23305800146319, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 29.198392850325618 } }, "raw_on_A": { "calibration": 83.18850868556241, "capability": 62.528725908581364, "intelligence": 41.86894313160032 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 34.17198402783528, "Emphasis on Accuracy 60:20:20": 29.237864520407634, "Emphasis on Cost 20:20:60": 34.662048899004674, "Emphasis on Speed 20:60:20": 40.422015539591015, "Intelligence only": 24.032711498388675 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0", "repo": "https://github.com/metask-ai/metask-jev", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": "Qwen3.5-4B merged r16 LoRA, candidate-logit readout", "adapter_id": "images/metask-jev-4b/spec.json:metask_v15.MetaskJevWarm", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7493261455525606, "acc_among_decisive": 0.8776978417266187, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "krishna765", "axes": { "calibration": 88.59116686443139, "cost": 63.66277849512568, "intelligence": 38.62465958342338, "speed": 89.93848373337222 }, "calibration": { "parts": { "choice": { "ece_hard": 0.06995357456541221, "ece_score": 86.00928508691756, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 86.00928508691756 }, "noul": { "abstention_rate": 0.3207547169811321, "brier": 0.11609579904406445, "ece": 0.05066343452944609, "n": 371, "score": 89.86731309411078 }, "score": { "interval90_coverage": 0.9539295392953929, "interval90_width": 0.47515808491418265, "n": 369, "nrps": 0.10083978781514244, "score": 89.89690241226586, "top_ece": 0.050611081969770266 } }, "score": 88.59116686443139 }, "capability": 63.60791322392738, "composite_ci95": { "A": [ 23.98211286851183, 50.42175552568154 ], "B": [ 20.76083692137617, 45.85216931192029 ], "C": [ 16.65424504757766, 35.01510800394553 ] }, "cost": { "basis": "ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): DeepInfra Qwen/Qwen3.5-4B list price $0.03/M input, $0.15/M output (base of JevK5, the adapter's base checkpoint; not on OpenRouter; frozen 25 Sep snapshot per I-3, same reference as reflex-4b, jobe, typecastlm and Quyet) x the server's own measured usage.input_tokens; logit readout, 0 output tokens; FP8 weight-only on 1x RTX PRO 6000 96GB (Lium; v1.6 GPU-class deviation from source H100), deck-4b 1.0.0 + jevk5 f944fe37; estimated, not charged", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0162645 }, "display": "deck-4B v1.0 (krishna765, JevK5 + LoRA, FP8 weight-only)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 38.79947935146771, "I_sealed": 38.44983981537905, "base": 38.62465958342338, "excess": -2.2237481063495843, "gap": 0.3496395360886595, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 58.68740862543909, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675674, "hard": 60.36489462094999, "judge": 64.40342781806196, "standard": 39.723733779824194 } }, "open|noul": { "cc": 25.017253278122844, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 39.13043478260869, "judge": 0.0, "standard": 11.111111111111116 } }, "open|score": { "cc": 32.6937761508412, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 51.877671536622636, "hard": 34.178023262130495, "judge": 35.59214965227232, "standard": 15.785773983225237 } }, "sealed|choice": { "cc": 59.157265682164855, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 69.40359175227256, "hard": 56.22171945701356, "judge": 59.031657355679705, "standard": 60.09360758714129 } }, "sealed|noul": { "cc": 21.704287246216186, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 48.0, "hard": 0.9708737864077666, "judge": 31.578947368421062, "standard": 35.2112676056338 } }, "sealed|score": { "cc": 34.48796651775611, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 66.00896406653321, "hard": 26.22895515401159, "judge": 35.59993017640196, "standard": 33.57754498288781 } } } }, "jevbench_score": 37.29399798821111, "key": "deck-4b-v1-0", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 20, "ranked": true, "ranks": { "A": 20, "B": 20, "C": 20, "capability": 26 }, "scores": { "A": 37.29399798821111, "B": 33.191383692005594, "C": 25.89860971403549 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.30241732355207207, "p50_s_raw": 0.07620866177603602, "p95_s_adjusted": 0.3353860241360962, "p95_s_raw": 0.09269301206804811 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 82.64482190603961, 89.1506890221124 ], "I": [ 32.29690931230169, 43.8177901770495 ], "capability": [ 58.30292793082468, 65.68855559803174 ], "composite": { "A": [ 23.98211286851183, 50.42175552568154 ], "B": [ 20.76083692137617, 45.85216931192029 ], "C": [ 16.65424504757766, 35.01510800394553 ] }, "seed": 33 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 88.59116686443139, "capability": 63.60791322392738, "intelligence": 38.62465958342338 }, "per_type": { "choice": { "accuracy": 0.688, "calibration": 86.00928508691756, "cc": 60.099656129515054, "cc_by_type": { "choice": 60.099656129515054 }, "cc_tiered": 58.922337153801976, "cc_tiered_open": 58.68740862543909, "cc_tiered_sealed": 59.157265682164855, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 60.099656129515054 }, "noul": { "accuracy": 0.6266666666666667, "calibration": 89.86731309411078, "cc": 25.333333333333343, "cc_by_type": { "noul": 25.333333333333343 }, "cc_tiered": 23.360770262169517, "cc_tiered_open": 25.017253278122844, "cc_tiered_sealed": 21.704287246216186, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 25.333333333333343 }, "score": { "accuracy": 0.5546666666666666, "calibration": 89.89690241226586, "cc": 35.52454801522435, "cc_by_type": { "score": 35.52454801522435 }, "cc_tiered": 33.590871334298654, "cc_tiered_open": 32.6937761508412, "cc_tiered_sealed": 34.48796651775611, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 35.52454801522435 } }, "raw_on_A": { "calibration": 89.0005990160775, "capability": 66.05004010202693, "intelligence": 43.09948118797635 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 33.95960169128873, "Emphasis on Accuracy 60:20:20": 28.553229571667234, "Emphasis on Cost 20:20:60": 35.46476300764428, "Emphasis on Speed 20:60:20": 39.807414011248, "Intelligence only": 23.04910072446208 }, "model_pin": null, "last_measured_on": "2026-10-05", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6792452830188679, "acc_among_decisive": 0.9325396825396826, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": null }, "repo": "https://github.com/krishna-gogineni-765/deck" }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Cloudflare", "axes": { "calibration": 85.46479352633332, "cost": 46.79963593205177, "intelligence": 42.18368888957246, "speed": 88.4044196893156 }, "calibration": { "parts": { "choice": { "ece_hard": 0.1119512500000001, "ece_score": 77.60974999999998, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 77.60974999999998 }, "noul": { "abstention_rate": 0.31733333333333336, "brier": 0.11674051933333338, "ece": 0.04965040000000006, "n": 375, "score": 90.06992 }, "score": { "interval90_coverage": 0.9333333333333333, "interval90_width": 0.460888888888889, "n": 375, "nrps": 0.09493298842, "score": 88.71471057900001, "top_ece": 0.06538639999999996 } }, "score": 85.46479352633332 }, "capability": 63.824241207952895, "composite_ci95": { "A": [ 26.640037750072654, 45.22280470107504 ], "B": [ 24.191496498186186, 42.57204972938278 ], "C": [ 18.500026215328234, 31.40472548685768 ] }, "cost": { "basis": "latest published v1.5 row (v1.5.8 combined file) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied; self-hosted estimate, not measured hosted billing", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.05933971674876847 }, "display": "Clef-Flash (Cloudflare, Qwen3.5-9B post-train with a joint schema head, multimodal, measured on text)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 45.25550449770323, "I_sealed": 39.11187328144169, "base": 42.18368888957246, "excess": 3.5702435738232956, "gap": 6.143631216261539, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 62.93462232292448, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 88.32432432432431, "hard": 56.59012268008808, "judge": 68.35860250494396, "standard": 54.79280033486815 } }, "open|noul": { "cc": 30.42788129744652, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 30.434782608695656, "judge": 0.0, "standard": 55.55555555555556 } }, "open|score": { "cc": 42.4040098727387, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 41.5230303030303, "hard": 37.280925925925935, "judge": 43.06322033898306, "standard": 52.101851851851855 } }, "sealed|choice": { "cc": 54.80150596378907, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 65.41275589387332, "hard": 45.53167420814478, "judge": 53.44506517690876, "standard": 70.07020569035596 } }, "sealed|noul": { "cc": 20.824374761599753, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 52.0, "hard": -10.679611650485432, "judge": 31.578947368421062, "standard": 52.112676056338024 } }, "sealed|score": { "cc": 41.70973911893625, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 54.71716312056738, "hard": 35.80301470588234, "judge": 41.72143712574849, "standard": 47.00192893401015 } } } }, "jevbench_score": 36.635050479655426, "key": "clef-flash", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 21, "ranked": true, "ranks": { "A": 21, "B": 18, "C": 21, "capability": 25 }, "scores": { "A": 36.635050479655426, "B": 33.967249202727025, "C": 25.44100727753849 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3556603590026498, "p50_s_raw": 0.10283017950132489, "p95_s_adjusted": 0.405996760725975, "p95_s_raw": 0.12799838036298752 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 79.4975513933287, 86.84610174167315 ], "I": [ 36.95680839342918, 46.203587775711 ], "capability": [ 59.38252934298857, 65.53439425442862 ], "composite": { "A": [ 26.640037750072654, 45.22280470107504 ], "B": [ 24.191496498186186, 42.57204972938278 ], "C": [ 18.500026215328234, 31.40472548685768 ] }, "seed": 21 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 85.46479352633332, "capability": 63.824241207952895, "intelligence": 42.18368888957246 }, "per_type": { "choice": { "accuracy": 0.6666666666666666, "calibration": 77.60974999999998, "cc": 57.37142748879814, "cc_by_type": { "choice": 57.37142748879814 }, "cc_tiered": 58.868064143356776, "cc_tiered_open": 62.93462232292448, "cc_tiered_sealed": 54.80150596378907, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 57.37142748879814 }, "noul": { "accuracy": 0.6373333333333333, "calibration": 90.06992, "cc": 27.46666666666666, "cc_by_type": { "noul": 27.46666666666666 }, "cc_tiered": 25.626128029523137, "cc_tiered_open": 30.42788129744652, "cc_tiered_sealed": 20.824374761599753, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 27.46666666666666 }, "score": { "accuracy": 0.6026666666666667, "calibration": 88.71471057900001, "cc": 42.01200877192982, "cc_by_type": { "score": 42.01200877192982 }, "cc_tiered": 42.05687449583748, "cc_tiered_open": 42.4040098727387, "cc_tiered_sealed": 41.70973911893625, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 42.01200877192982 } }, "raw_on_A": { "calibration": 82.15484704393162, "capability": 63.82820945816345, "intelligence": 45.501571872395274 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 33.178006784894365, "Emphasis on Accuracy 60:20:20": 30.03860559786673, "Emphasis on Cost 20:20:60": 31.455768602881705, "Emphasis on Speed 20:60:20": 39.463018165391404, "Intelligence only": 26.30501767147255 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/Cloudflare/clef-flash", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100 80GB), offline (HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE), credential-free, server bound to 127.0.0.1, harness on the same pod over loopback HTTP; Lium pod 08a1ff4a-248a-4ff2-948e-ea558f99e808 (huid cosmic-hawk-4f, node calm-fox-65), 1x NVIDIA H100 80GB HBM3, United States, USD 1.30/h", "underlying": "Qwen/Qwen3.5-9B", "adapter_id": "/home/flori/jobs/jevbench-add-requests-20260919/r39/outputs/meta/specs/clef-flash.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "alt": { "label": "Workers AI price scenario (latency unmeasured)", "note": "Cloudflare standard launch input price; same self-hosted accuracy and adjusted H100 latency held fixed for this price-only scenario. No Workers AI latency was measured; this is not a hosted API result and does not set official Capability eligibility. Official self-hosted row uses the base-model reference price.", "usd_per_1000": 0.053405745073891626, "axes": { "intelligence": 53.08161490373422, "calibration": 87.5533640278553, "speed": 83.63107135213085, "cost": 48.17236064887202 }, "pricing_only": true }, "model_pin": "Cloudflare/clef-flash@17f0b0ad64efb65d273590632833508766b2aae6", "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6826666666666666, "acc_among_decisive": 0.93359375, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "jsaurabh", "axes": { "calibration": 79.78895435560509, "cost": 45.690218065856605, "intelligence": 41.94184606895763, "speed": 85.40793837616167 }, "calibration": { "parts": { "choice": { "ece_hard": 0.14238786731364284, "ece_score": 71.52242653727143, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 71.52242653727143 }, "noul": { "abstention_rate": 0.2911051212938005, "brier": 0.12127289774594222, "ece": 0.07874545275117417, "n": 371, "score": 84.25090944976516 }, "score": { "interval90_coverage": 0.8888888888888888, "interval90_width": 0.31820234869015335, "n": 369, "nrps": 0.0937862889000391, "score": 83.59352707977865, "top_ece": 0.11717158475219394 } }, "score": 79.78895435560509 }, "capability": 60.86540021228136, "composite_ci95": { "A": [ 22.862450507726937, 41.74491188365231 ], "B": [ 20.753639028685242, 39.453425031751074 ], "C": [ 15.87670174147704, 28.989522141425212 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.06461391625615764 }, "display": "Qwen3.5-9B Jev-like data-mix v2", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 45.69949425428299, "I_sealed": 38.184197883632265, "base": 41.94184606895763, "excess": 4.94190872821248, "gap": 7.515296370650724, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 56.305297450788785, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675674, "hard": 49.040578798364265, "judge": 56.49307844429796, "standard": 62.32733361239012 } }, "open|noul": { "cc": 33.4120082815735, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 30.434782608695656, "judge": 19.999999999999996, "standard": 33.33333333333333 } }, "open|score": { "cc": 47.38117703048668, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 50.73251602617265, "hard": 54.043486375775075, "judge": 35.50551913484291, "standard": 50.19437568553258 } }, "sealed|choice": { "cc": 54.756843153610475, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 64.08247727440693, "hard": 46.04072398190045, "judge": 51.58286778398511, "standard": 72.28722749107035 } }, "sealed|noul": { "cc": 20.448763917177054, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 52.0, "hard": -4.854368932038833, "judge": 26.315789473684205, "standard": 46.47887323943662 } }, "sealed|score": { "cc": 39.346986580109274, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 69.00077177598622, "hard": 33.3124721590493, "judge": 38.59270672300753, "standard": 37.720542609943344 } } } }, "jevbench_score": 33.58966697347588, "key": "qwen35-9b-jev-data-mix-v2", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 22, "ranked": true, "ranks": { "A": 22, "B": 22, "C": 22, "capability": 35 }, "scores": { "A": 33.58966697347588, "B": 31.31610274190126, "C": 23.326157620469353 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.4865717590786517, "p50_s_raw": 0.16828587953932583, "p95_s_adjusted": 0.5916423641145229, "p95_s_raw": 0.22082118205726145 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 75.15659713819453, 82.50836404329671 ], "I": [ 35.70342834084104, 45.9106569165651 ], "capability": [ 56.2798648487553, 63.69018289459294 ], "composite": { "A": [ 22.862450507726937, 41.74491188365231 ], "B": [ 20.753639028685242, 39.453425031751074 ], "C": [ 15.87670174147704, 28.989522141425212 ] }, "seed": 92 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 79.78895435560509, "capability": 60.86540021228136, "intelligence": 41.94184606895763 }, "per_type": { "choice": { "accuracy": 0.6533333333333333, "calibration": 71.52242653727143, "cc": 55.666284588350074, "cc_by_type": { "choice": 55.666284588350074 }, "cc_tiered": 55.53107030219963, "cc_tiered_open": 56.305297450788785, "cc_tiered_sealed": 54.756843153610475, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 55.666284588350074 }, "noul": { "accuracy": 0.6373333333333333, "calibration": 84.25090944976516, "cc": 27.46666666666666, "cc_by_type": { "noul": 27.46666666666666 }, "cc_tiered": 26.930386099375276, "cc_tiered_open": 33.4120082815735, "cc_tiered_sealed": 20.448763917177054, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 27.46666666666666 }, "score": { "accuracy": 0.5733333333333334, "calibration": 83.59352707977865, "cc": 42.24339448080683, "cc_by_type": { "score": 42.24339448080683 }, "cc_tiered": 43.36408180529798, "cc_tiered_open": 47.38117703048668, "cc_tiered_sealed": 39.346986580109274, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 42.24339448080683 } }, "raw_on_A": { "calibration": 80.51953554017331, "capability": 63.66543855723072, "intelligence": 46.811341574288136 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 30.68928023743452, "Emphasis on Accuracy 60:20:20": 27.947003437168036, "Emphasis on Cost 20:20:60": 29.02722060908932, "Emphasis on Speed 20:60:20": 36.335136837664905, "Intelligence only": 24.643879144700183 }, "class": "jev-rebuild", "open": "unknown", "licence": "Apache-2.0 (adapter code and weights; Qwen3.5-9B base is Apache-2.0)", "repo": "https://huggingface.co/jsaurabh/qwen3.5-9b-jev-data-mix-v2", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": null, "adapter_id": "images/qwen35-9b-jev-data-mix-v2/spec.json:qwen35_nimble_lora.Qwen35NimbleLoraAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7088948787061995, "acc_among_decisive": 0.908745247148289, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "allebee", "axes": { "calibration": 89.38710082819252, "cost": 63.07359395281681, "intelligence": 36.261123811123895, "speed": 91.56570599843724 }, "calibration": { "parts": { "choice": { "ece_hard": 0.05722074836783767, "ece_score": 88.55585032643246, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 88.55585032643246 }, "noul": { "abstention_rate": 0.3800539083557951, "brier": 0.12953478188668033, "ece": 0.0506873602652232, "n": 371, "score": 89.86252794695537 }, "score": { "interval90_coverage": 0.981029810298103, "interval90_width": 0.5517163504968384, "n": 369, "nrps": 0.098178280103309, "score": 89.74292421118973, "top_ece": 0.053481617836448225 } }, "score": 89.38710082819252 }, "capability": 62.824112319658205, "composite_ci95": { "A": [ 21.332485805393638, 41.48362112591029 ], "B": [ 18.26634005992914, 37.259117524916284 ], "C": [ 14.814226253745582, 28.808070226326592 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01701689039408867 }, "display": "JevK5 v0.2.0", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 39.85943426682522, "I_sealed": 32.66281335542257, "base": 36.261123811123895, "excess": 4.623233268964404, "gap": 7.196620911402647, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 56.2584494703768, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 57.189189189189186, "hard": 52.815350739226176, "judge": 56.49307844429796, "standard": 62.32733361239012 } }, "open|noul": { "cc": 29.9399585921325, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 47.82608695652173, "judge": -9.999999999999998, "standard": 33.33333333333333 } }, "open|score": { "cc": 33.37989473796636, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 58.905656531340256, "hard": 27.673703730451283, "judge": 40.84579963079196, "standard": 20.83053851707115 } }, "sealed|choice": { "cc": 55.096468823579634, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 64.08247727440693, "hard": 55.20361990950226, "judge": 51.58286778398511, "standard": 55.65956398571254 } }, "sealed|noul": { "cc": 6.636284338632712, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 43.99999999999999, "hard": -14.563106796116509, "judge": 18.421052631578938, "standard": 12.676056338028175 } }, "sealed|score": { "cc": 36.25568690405537, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 64.58927438023056, "hard": 27.57268337567902, "judge": 41.004484216055225, "standard": 32.331704254720705 } } } }, "jevbench_score": 32.09901620353128, "key": "jevk5-v02", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 23, "ranked": true, "ranks": { "A": 23, "B": 24, "C": 23, "capability": 27 }, "scores": { "A": 32.09901620353128, "B": 28.24080105110956, "C": 22.290983474674498 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.23078905596630647, "p50_s_raw": 0.04039452798315324, "p95_s_adjusted": 0.30214414960937575, "p95_s_raw": 0.07607207480468788 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 83.43880788393268, 90.09857764344801 ], "I": [ 30.721600512376973, 40.44080848333523 ], "capability": [ 58.00843038572983, 64.2177747361447 ], "composite": { "A": [ 21.332485805393638, 41.48362112591029 ], "B": [ 18.26634005992914, 37.259117524916284 ], "C": [ 14.814226253745582, 28.808070226326592 ] }, "seed": 49 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.38710082819252, "capability": 62.824112319658205, "intelligence": 36.261123811123895 }, "per_type": { "choice": { "accuracy": 0.6546666666666666, "calibration": 88.55585032643246, "cc": 55.836798878394866, "cc_by_type": { "choice": 55.836798878394866 }, "cc_tiered": 55.67745914697822, "cc_tiered_open": 56.2584494703768, "cc_tiered_sealed": 55.096468823579634, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 55.836798878394866 }, "noul": { "accuracy": 0.5733333333333334, "calibration": 89.86252794695537, "cc": 14.666666666666671, "cc_by_type": { "noul": 14.666666666666671 }, "cc_tiered": 18.288121465382606, "cc_tiered_open": 29.9399585921325, "cc_tiered_sealed": 6.636284338632712, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 14.666666666666671 }, "score": { "accuracy": 0.5546666666666666, "calibration": 89.74292421118973, "cc": 36.384921485746844, "cc_by_type": { "score": 36.384921485746844 }, "cc_tiered": 34.81779082101087, "cc_tiered_open": 33.37989473796636, "cc_tiered_sealed": 36.25568690405537, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 36.384921485746844 } }, "raw_on_A": { "calibration": 85.60375573005521, "capability": 62.620633576812004, "intelligence": 39.637511423568796 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 29.02935442351159, "Emphasis on Accuracy 60:20:20": 24.01393367218798, "Emphasis on Cost 20:20:60": 30.556198079902455, "Emphasis on Speed 20:60:20": 34.51299399305244, "Intelligence only": 19.071452492870637 }, "class": "jev-rebuild", "open": "unknown", "licence": "Apache-2.0 (code/adapter); Apache-2.0 (Qwen3.5-4B base)", "repo": "https://github.com/allebee/jevk5", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": null, "adapter_id": "images/jevk5-v02/spec.json:jevk5_direct.JevK5DirectAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6199460916442049, "acc_among_decisive": 0.9347826086956522, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Juspay", "axes": { "calibration": 91.76581828544445, "cost": 39.84212731591276, "intelligence": 45.955077428728934, "speed": 89.83885080776159 }, "calibration": { "parts": { "choice": { "ece_hard": 0.034452812499999874, "ece_score": 93.10943750000003, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 93.10943750000003 }, "noul": { "abstention_rate": 0.376, "brier": 0.10284091864000006, "ece": 0.03908880000000003, "n": 375, "score": 92.18224 }, "score": { "interval90_coverage": 0.9706666666666667, "interval90_width": 0.3935555555555556, "n": 375, "nrps": 0.08401831953999996, "score": 90.00577735633334, "top_ece": 0.05793306666666674 } }, "score": 91.76581828544445 }, "capability": 68.8604478570867, "composite_ci95": { "A": [ 23.186488200616, 37.838876804389514 ], "B": [ 21.60212409324276, 36.562829588228304 ], "C": [ 16.101727917094447, 27.301863296622916 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.10121908866995075 }, "display": "JevOne", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 47.580446331109215, "I_sealed": 44.329708526348654, "base": 45.955077428728934, "excess": 0.6773501623223175, "gap": 3.2507378047605613, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 80.11686099440308, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 84.43243243243244, "hard": 77.35136835482857, "judge": 84.17930125247197, "standard": 77.39640016743407 } }, "open|noul": { "cc": 20.761214630779854, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 30.434782608695656, "judge": -9.999999999999998, "standard": 22.222222222222232 } }, "open|score": { "cc": 41.86326336814473, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 53.82060606060606, "hard": 35.2175, "judge": 43.91203389830509, "standard": 46.102962962962955 } }, "sealed|choice": { "cc": 71.7077246661197, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 82.70637794693668, "hard": 65.89366515837104, "judge": 73.9292364990689, "standard": 74.50424929178472 } }, "sealed|noul": { "cc": 13.421236874491711, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 48.0, "hard": -8.737864077669899, "judge": 13.157894736842103, "standard": 40.8450704225352 } }, "sealed|score": { "cc": 47.86016403843456, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 63.93652482269504, "hard": 43.95980392156864, "judge": 49.40958083832332, "standard": 45.29857868020299 } } } }, "jevbench_score": 31.14510331640707, "key": "jevone", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 24, "ranked": true, "ranks": { "A": 24, "B": 23, "C": 24, "capability": 16 }, "scores": { "A": 31.14510331640707, "B": 29.58577784402432, "C": 21.62854396972713 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2735971907153726, "p50_s_raw": 0.06179859535768628, "p95_s_adjusted": 0.3793178545311093, "p95_s_raw": 0.11465892726555464 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 85.59503348121109, 91.11403729704442 ], "I": [ 40.574264452604204, 50.931916001637504 ], "capability": [ 63.974665483283545, 69.89087834774071 ], "composite": { "A": [ 23.186488200616, 37.838876804389514 ], "B": [ 21.60212409324276, 36.562829588228304 ], "C": [ 16.101727917094447, 27.301863296622916 ] }, "seed": 50 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 91.76581828544445, "capability": 68.8604478570867, "intelligence": 45.955077428728934 }, "per_type": { "choice": { "accuracy": 0.7946666666666666, "calibration": 93.10943750000003, "cc": 73.74079933309964, "cc_by_type": { "choice": 73.74079933309964 }, "cc_tiered": 75.91229283026139, "cc_tiered_open": 80.11686099440308, "cc_tiered_sealed": 71.7077246661197, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 73.74079933309964 }, "noul": { "accuracy": 0.5973333333333334, "calibration": 92.18224, "cc": 19.466666666666676, "cc_by_type": { "noul": 19.466666666666676 }, "cc_tiered": 17.091225752635783, "cc_tiered_open": 20.761214630779854, "cc_tiered_sealed": 13.421236874491711, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 19.466666666666676 }, "score": { "accuracy": 0.6, "calibration": 90.00577735633334, "cc": 46.965815789473666, "cc_by_type": { "score": 46.965815789473666 }, "cc_tiered": 44.861713703289645, "cc_tiered_open": 41.86326336814473, "cc_tiered_sealed": 47.86016403843456, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 46.965815789473666 } }, "raw_on_A": { "calibration": 86.03357993512108, "capability": 69.68729793470257, "intelligence": 53.34101593428407 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 27.74831039177889, "Emphasis on Accuracy 60:20:20": 26.4196991707309, "Emphasis on Cost 20:20:60": 24.78908517771933, "Emphasis on Speed 20:60:20": 33.418202927958156, "Intelligence only": 24.64935315350002 }, "class": "jev-rebuild", "open": "yes", "licence": "Component-specific JevOne/Qwen/SGLang terms recorded in RESULT.md", "repo": "https://huggingface.co/juspay/jev-one", "gpu": "RTXPRO6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container", "underlying": "Qwen3.6-35B-A3B BF16 with JevOne bidirectional option-logit mapping", "adapter_id": "images/jevone/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.624, "acc_among_decisive": 0.9572649572649573, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "HopitAI", "axes": { "calibration": 90.07902805935959, "cost": 62.25627699896333, "intelligence": 34.73970870443837, "speed": 91.11434937226889 }, "calibration": { "parts": { "choice": { "ece_hard": 0.054931646080717024, "ece_score": 89.0136707838566, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 89.0136707838566 }, "noul": { "abstention_rate": 0.43733333333333335, "brier": 0.1320524837128236, "ece": 0.03940530319869939, "n": 375, "score": 92.11893936026013 }, "score": { "interval90_coverage": 0.9733333333333334, "interval90_width": 0.6148888888888887, "n": 375, "nrps": 0.110200461495695, "score": 89.10447403396208, "top_ece": 0.053855028912531695 } }, "score": 90.07902805935959 }, "capability": 62.40936838189898, "composite_ci95": { "A": [ 19.749752505423817, 34.23513789747224 ], "B": [ 16.849888107534454, 30.36407576551454 ], "C": [ 13.715105906544313, 23.774401317689065 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.018118577586206896 }, "display": "Hopper", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 40.99371798867166, "I_sealed": 29.284169358941572, "base": 35.138943673806615, "excess": 9.136160987291847, "gap": 11.70954862973009, "penalty": 0.9886383901270815, "per_type_split": { "open|choice": { "cc": 68.98723802511984, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864865, "hard": 64.1396665618119, "judge": 72.31377719182596, "standard": 69.8618668899121 } }, "open|noul": { "cc": 18.554865424430645, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": 30.434782608695656, "judge": -19.999999999999996, "standard": 33.33333333333333 } }, "open|score": { "cc": 35.439050516464505, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 56.1505221295817, "hard": 30.729313281528604, "judge": 33.331159411757824, "standard": 37.66462583683774 } }, "sealed|choice": { "cc": 57.26398771697454, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 68.07331313280615, "hard": 49.60407239819003, "judge": 59.031657355679705, "standard": 64.52765118857003 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 56.00000000000001, "hard": -20.38834951456311, "judge": 0.0, "standard": -1.4084507042253502 } }, "sealed|score": { "cc": 30.58852035985018, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 62.587460120023806, "hard": 30.318479212532644, "judge": 26.893973849966592, "standard": 20.67095253922381 } } } }, "jevbench_score": 28.852652318563656, "key": "hopper", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 25, "ranked": true, "ranks": { "A": 25, "B": 25, "C": 25, "capability": 28 }, "scores": { "A": 28.852652318563656, "B": 25.218775497922103, "C": 20.036564110113652 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.26295550270006063, "p50_s_raw": 0.0564777513500303, "p95_s_adjusted": 0.2942271867766976, "p95_s_raw": 0.07211359338834882 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 83.58654904175489, 89.37856892562735 ], "I": [ 29.827996307138942, 37.38986029909742 ], "capability": [ 57.58520429455299, 62.579268195863094 ], "composite": { "A": [ 19.749752505423817, 34.23513789747224 ], "B": [ 16.849888107534454, 30.36407576551454 ], "C": [ 13.715105906544313, 23.774401317689065 ] }, "seed": 41 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 90.07902805935959, "capability": 62.40936838189898, "intelligence": 34.73970870443837 }, "per_type": { "choice": { "accuracy": 0.6866666666666666, "calibration": 89.0136707838566, "cc": 59.92914183947025, "cc_by_type": { "choice": 59.92914183947025 }, "cc_tiered": 63.12561287104719, "cc_tiered_open": 68.98723802511984, "cc_tiered_sealed": 57.26398771697454, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 59.92914183947025 }, "noul": { "accuracy": 0.5306666666666666, "calibration": 92.11893936026013, "cc": 6.133333333333324, "cc_by_type": { "noul": 6.133333333333324 }, "cc_tiered": 9.277432712215322, "cc_tiered_open": 18.554865424430645, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 6.133333333333324 }, "score": { "accuracy": 0.5066666666666667, "calibration": 89.10447403396208, "cc": 33.257420216120906, "cc_by_type": { "score": 33.257420216120906 }, "cc_tiered": 33.01378543815734, "cc_tiered_open": 35.439050516464505, "cc_tiered_sealed": 30.58852035985018, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 33.257420216120906 } }, "raw_on_A": { "calibration": 87.40904942545944, "capability": 64.64647397045381, "intelligence": 41.88389851544817 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 25.942845868326373, "Emphasis on Accuracy 60:20:20": 21.285843260250505, "Emphasis on Cost 20:20:60": 27.444368192058636, "Emphasis on Speed 20:60:20": 31.034805184083435, "Intelligence only": 16.77021030692692 }, "class": "jev-rebuild", "open": "yes", "licence": "Component-specific terms recorded in RESULT.md; submitted adapter release and Qwen base retain their respective terms", "repo": "https://huggingface.co/HopitAI/hopper", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen3.5-4B plus HopitAI/hopper LoRA", "adapter_id": "images/hopper/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5626666666666666, "acc_among_decisive": 0.943127962085308, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "unknown", "axes": { "calibration": 89.1627567462182, "cost": 63.26458220826236, "intelligence": 34.62714018309005, "speed": 90.51878000442085 }, "calibration": { "parts": { "choice": { "ece_hard": 0.04836725385924713, "ece_score": 90.32654922815058, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 90.32654922815058 }, "noul": { "abstention_rate": 0.477088948787062, "brier": 0.12273935128941174, "ece": 0.06563823281458846, "n": 371, "score": 86.87235343708231 }, "score": { "interval90_coverage": 0.967479674796748, "interval90_width": 0.43699186991869926, "n": 369, "nrps": 0.08429976663999325, "score": 90.28936757342169, "top_ece": 0.054956440945786535 } }, "score": 89.1627567462182 }, "capability": 61.89494846465412, "composite_ci95": { "A": [ 21.33216708823923, 38.39023119963257 ], "B": [ 18.266955184224454, 34.31649986004929 ], "C": [ 14.814004922388358, 26.659882777522615 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (I-2): measured input tokens; zero generated output tokens for signed logits readout. M2 floor uses the 25 Sep 2026 DeepInfra Qwen3.5-4B snapshot rates (USD 0.03/M input, USD 0.15/M output); the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen3.5-9B.", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.016769261083743846 }, "display": "Imajev-4B (RTX 5090)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 35.03758089292987, "I_sealed": 34.21669947325023, "base": 34.62714018309005, "excess": -1.752506222758607, "gap": 0.8208814196796368, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 66.47085555091229, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 80.54054054054055, "hard": 58.47750865051904, "judge": 80.22412656558997, "standard": 54.79280033486815 } }, "open|noul": { "cc": 0.6756383712905452, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": 4.347826086956519, "judge": -30.000000000000004, "standard": 11.111111111111116 } }, "open|score": { "cc": 37.966248756586765, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 52.45615279420953, "hard": 43.92313144450569, "judge": 33.905546551444175, "standard": 24.898584669651413 } }, "sealed|choice": { "cc": 62.38689563101728, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 70.73387037173899, "hard": 53.676470588235276, "judge": 64.61824953445065, "standard": 72.28722749107035 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 39.99999999999999, "hard": -30.097087378640776, "judge": 0.0, "standard": 7.042253521126751 } }, "sealed|score": { "cc": 40.26320278873341, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 63.176956160290466, "hard": 35.01688311771597, "judge": 46.365402771352585, "standard": 30.14566547106101 } } } }, "jevbench_score": 28.655751132526984, "key": "imajev-4b-rtx5090-a2", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 26, "ranked": true, "ranks": { "A": 26, "B": 26, "C": 26, "capability": 29 }, "scores": { "A": 28.655751132526984, "B": 25.0248976556334, "C": 19.899827175365957 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.29240404299926015, "p50_s_raw": 0.07120202149963006, "p95_s_adjusted": 0.3034859753679484, "p95_s_raw": 0.07674298768397421 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 83.2060903767957, 89.78727704768245 ], "I": [ 30.707360525579823, 39.19772341159948 ], "capability": [ 58.11557152205388, 63.38857993223046 ], "composite": { "A": [ 21.33216708823923, 38.39023119963257 ], "B": [ 18.266955184224454, 34.31649986004929 ], "C": [ 14.814004922388358, 26.659882777522615 ] }, "seed": 42 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.1627567462182, "capability": 61.89494846465412, "intelligence": 34.62714018309005 }, "per_type": { "choice": { "accuracy": 0.7146666666666667, "calibration": 90.32654922815058, "cc": 63.50994193041121, "cc_by_type": { "choice": 63.50994193041121 }, "cc_tiered": 64.4288755909648, "cc_tiered_open": 66.47085555091229, "cc_tiered_sealed": 62.38689563101728, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 63.50994193041121 }, "noul": { "accuracy": 0.49866666666666665, "calibration": 86.87235343708231, "cc": -0.2666666666666706, "cc_by_type": { "noul": -0.2666666666666706 }, "cc_tiered": 0.3378191856452726, "cc_tiered_open": 0.6756383712905452, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.56, "calibration": 90.28936757342169, "cc": 40.37192786146962, "cc_by_type": { "score": 40.37192786146962 }, "cc_tiered": 39.11472577266009, "cc_tiered_open": 37.966248756586765, "cc_tiered_sealed": 40.26320278873341, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 40.37192786146962 } }, "raw_on_A": { "calibration": 87.91523763851959, "capability": 61.35771088116867, "intelligence": 34.80018412381776 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 25.816712856140082, "Emphasis on Accuracy 60:20:20": 21.130055842968158, "Emphasis on Cost 20:20:60": 27.454793753723415, "Emphasis on Speed 20:60:20": 30.81252811725961, "Intelligence only": 16.607714361099795 }, "class": "unclassified", "open": "unknown", "licence": "not recorded", "repo": "https://github.com/fstandhartinger/jevbench/issues/110", "gpu": "RTX 5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX 5090), offline read-only container", "underlying": null, "adapter_id": "A2 reviewed bundle", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.522911051212938, "acc_among_decisive": 0.9639175257731959, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "newfull5 (dhtocks)", "axes": { "calibration": 88.43741591513763, "cost": 55.82433433950372, "intelligence": 33.840703805987175, "speed": 89.51571180654442 }, "calibration": { "parts": { "choice": { "ece_hard": 0.0565084690553745, "ece_score": 88.6983061889251, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 88.6983061889251 }, "noul": { "abstention_rate": 0.3504043126684636, "brier": 0.13494345407008085, "ece": 0.06141401617250676, "n": 371, "score": 87.71719676549866 }, "score": { "interval90_coverage": 0.926829268292683, "interval90_width": 0.5194218608852753, "n": 369, "nrps": 0.1123930174593495, "score": 88.89674479098915, "top_ece": 0.054836043360433705 } }, "score": 88.43741591513763 }, "capability": 61.139059860562405, "composite_ci95": { "A": [ 17.563735957847147, 34.57907875172623 ], "B": [ 15.042433721702132, 31.023588122626016 ], "C": [ 12.197038859616075, 24.013249133143216 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.02968398399014778 }, "display": "Malkuth-4B (newfull5, Kev post-train)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 36.78555463394936, "I_sealed": 30.895852978024994, "base": 33.840703805987175, "excess": 3.316314013486122, "gap": 5.889701655924366, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 57.213904436116636, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 68.86486486486487, "hard": 47.153192827933324, "judge": 68.35860250494396, "standard": 54.79280033486815 } }, "open|noul": { "cc": 19.189786059351285, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 30.434782608695656, "judge": -19.999999999999996, "standard": 22.222222222222232 } }, "open|score": { "cc": 33.95297340638018, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 51.45848484848485, "hard": 32.323055555555534, "judge": 29.900169491525418, "standard": 34.53925925925927 } }, "sealed|choice": { "cc": 53.38404633359997, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 69.40359175227256, "hard": 43.49547511312216, "judge": 51.58286778398511, "standard": 67.8531838896416 } }, "sealed|noul": { "cc": 12.571668333969063, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 48.0, "hard": -16.504854368932044, "judge": 26.315789473684205, "standard": 32.3943661971831 } }, "sealed|score": { "cc": 26.73184426650596, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 63.09113475177305, "hard": 12.109877450980411, "judge": 28.95095808383231, "standard": 34.467461928933965 } } } }, "jevbench_score": 26.197476126697616, "key": "malkuth-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 27, "ranked": true, "ranks": { "A": 27, "B": 28, "C": 27, "capability": 34 }, "scores": { "A": 26.197476126697616, "B": 23.020714347323178, "C": 18.19269175465112 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2985552769736387, "p50_s_raw": 0.07427763848681934, "p95_s_adjusted": 0.3744588227418717, "p95_s_raw": 0.11222941137093585 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 82.33637885013427, 89.07368375555646 ], "I": [ 28.800746942905594, 38.02075761880322 ], "capability": [ 56.551766449658025, 62.63523991792939 ], "composite": { "A": [ 17.563735957847147, 34.57907875172623 ], "B": [ 15.042433721702132, 31.023588122626016 ], "C": [ 12.197038859616075, 24.013249133143216 ] }, "seed": 64 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 88.43741591513763, "capability": 61.139059860562405, "intelligence": 33.840703805987175 }, "per_type": { "choice": { "accuracy": 0.6466666666666666, "calibration": 88.6983061889251, "cc": 54.813713138126026, "cc_by_type": { "choice": 54.813713138126026 }, "cc_tiered": 55.2989753848583, "cc_tiered_open": 57.213904436116636, "cc_tiered_sealed": 53.38404633359997, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 54.813713138126026 }, "noul": { "accuracy": 0.592, "calibration": 87.71719676549866, "cc": 18.399999999999995, "cc_by_type": { "noul": 18.399999999999995 }, "cc_tiered": 15.880727196660175, "cc_tiered_open": 19.189786059351285, "cc_tiered_sealed": 12.571668333969063, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 18.399999999999995 }, "score": { "accuracy": 0.5173333333333333, "calibration": 88.89674479098915, "cc": 29.251991228070175, "cc_by_type": { "score": 29.251991228070175 }, "cc_tiered": 30.34240883644307, "cc_tiered_open": 33.95297340638018, "cc_tiered_sealed": 26.73184426650596, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 29.251991228070175 } }, "raw_on_A": { "calibration": 85.35554104202156, "capability": 62.08736671102251, "intelligence": 38.819192380023466 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 23.437149264983223, "Emphasis on Accuracy 60:20:20": 19.45371080344073, "Emphasis on Cost 20:20:60": 24.246787312645555, "Emphasis on Speed 20:60:20": 28.28433154980761, "Intelligence only": 15.501658014110385 }, "class": "jev-rebuild", "open": "yes", "licence": "CC-BY-NC-4.0, research use only (XNLI and RACE in the training mix)", "repo": "https://github.com/newfull5/malkuth", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": null, "adapter_id": "images/malkuth-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6495956873315364, "acc_among_decisive": 0.921161825726141, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "hjmurmur (Octalab)", "axes": { "calibration": 89.87138320201578, "cost": 51.172592607797476, "intelligence": 33.85815624201783, "speed": 83.1032423977654 }, "calibration": { "parts": { "choice": { "ece_hard": 0.06281808353596482, "ece_score": 87.43638329280702, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 87.43638329280702 }, "noul": { "abstention_rate": 0.34231805929919135, "brier": 0.11806081980479606, "ece": 0.04224066622806287, "n": 371, "score": 91.55186675438742 }, "score": { "interval90_coverage": 0.959349593495935, "interval90_width": 0.5268744354110209, "n": 369, "nrps": 0.10409415210041804, "score": 90.6258995588529, "top_ece": 0.04169392836126208 } }, "score": 89.87138320201578 }, "capability": 61.8647697220168, "composite_ci95": { "A": [ 15.505551311407853, 35.88495273799785 ], "B": [ 13.30350456735459, 32.67206876334683 ], "C": [ 10.767743966255452, 24.920106068054068 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.042420935960591134 }, "display": "jqv (Qwen3-32B zero-shot)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 33.07530766335527, "I_sealed": 34.641004820680386, "base": 33.85815624201783, "excess": -4.139084799763362, "gap": -1.5656971573251184, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 58.31263706393481, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864865, "hard": 47.153192827933324, "judge": 64.40342781806196, "standard": 62.32733361239012 } }, "open|noul": { "cc": 11.77087646652864, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 4.347826086956519, "judge": -19.999999999999996, "standard": 44.44444444444444 } }, "open|score": { "cc": 29.142409459602362, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 31.012596729013953, "hard": 34.8410871612874, "judge": 43.10234913255198, "standard": -4.129949087897944 } }, "sealed|choice": { "cc": 53.61923821279086, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 62.75219865494053, "hard": 49.60407239819003, "judge": 55.30726256983241, "standard": 54.55105308535534 } }, "sealed|noul": { "cc": 17.979675703685576, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 60.00000000000001, "hard": -4.854368932038833, "judge": 21.052631578947366, "standard": 38.0281690140845 } }, "sealed|score": { "cc": 32.32410054556472, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 52.65739446915423, "hard": 22.31408478189991, "judge": 37.34025956092481, "standard": 34.65324658805946 } } } }, "jevbench_score": 25.391439220985372, "key": "jqv", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 28, "ranked": true, "ranks": { "A": 28, "B": 29, "C": 29, "capability": 30 }, "scores": { "A": 25.391439220985372, "B": 22.528313466257792, "C": 17.63294390346206 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3448032220825553, "p50_s_raw": 0.09740161104127765, "p95_s_adjusted": 1.4193988358601928, "p95_s_raw": 0.6346994179300964 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 83.42862757082423, 89.91697762368707 ], "I": [ 27.681410439375405, 39.196898115223256 ], "capability": [ 56.87716097755697, 63.63771518645792 ], "composite": { "A": [ 15.505551311407853, 35.88495273799785 ], "B": [ 13.30350456735459, 32.67206876334683 ], "C": [ 10.767743966255452, 24.920106068054068 ] }, "seed": 51 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.87138320201578, "capability": 61.8647697220168, "intelligence": 33.85815624201783 }, "per_type": { "choice": { "accuracy": 0.648, "calibration": 87.43638329280702, "cc": 54.98422742817084, "cc_by_type": { "choice": 54.98422742817084 }, "cc_tiered": 55.96593763836283, "cc_tiered_open": 58.31263706393481, "cc_tiered_sealed": 53.61923821279086, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 54.98422742817084 }, "noul": { "accuracy": 0.6106666666666667, "calibration": 91.55186675438742, "cc": 22.13333333333334, "cc_by_type": { "noul": 22.13333333333334 }, "cc_tiered": 14.875276085107107, "cc_tiered_open": 11.77087646652864, "cc_tiered_sealed": 17.979675703685576, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 22.13333333333334 }, "score": { "accuracy": 0.5066666666666667, "calibration": 90.6258995588529, "cc": 32.28871510053675, "cc_by_type": { "score": 32.28871510053675 }, "cc_tiered": 30.73325500258354, "cc_tiered_open": 29.142409459602362, "cc_tiered_sealed": 32.32410054556472, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 32.28871510053675 } }, "raw_on_A": { "calibration": 85.0933345598925, "capability": 62.946394387426395, "intelligence": 40.79945421496029 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 22.511071395625475, "Emphasis on Accuracy 60:20:20": 19.077648593308684, "Emphasis on Cost 20:20:60": 22.883252890033052, "Emphasis on Speed 20:60:20": 26.917672025192456, "Intelligence only": 15.525654079176814 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 (Qwen3-32B weights); serving code public", "repo": "https://github.com/Octalab-Inc/jqv", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "Qwen/Qwen3-32B, bf16, read as a direct-logit classifier (no fine-tuning)", "adapter_id": "images/jqv/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6576819407008087, "acc_among_decisive": 0.9385245901639344, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Mapika", "axes": { "calibration": 83.61830413355554, "cost": 34.38579876859529, "intelligence": 48.72471958402083, "speed": 91.27357492980725 }, "calibration": { "parts": { "choice": { "ece_hard": 0.10659875000000013, "ece_score": 78.68024999999997, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 78.68024999999997 }, "noul": { "abstention_rate": 0.2613333333333333, "brier": 0.11783857269333342, "ece": 0.07045920000000003, "n": 375, "score": 85.90816 }, "score": { "interval90_coverage": 0.9253333333333333, "interval90_width": 0.40466666666666656, "n": 375, "nrps": 0.08455155198666667, "score": 86.26650240066667, "top_ece": 0.09505919999999997 } }, "score": 83.61830413355554 }, "capability": 66.1715118587882, "composite_ci95": { "A": [ 18.69889165560131, 26.594050353427875 ], "B": [ 17.887370941224734, 26.251970770447716 ], "C": [ 12.985341427500913, 20.369267436681472 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.1538650708128079 }, "display": "decider-35b-a3b (Mapika)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 52.787614448504826, "I_sealed": 44.66182471953684, "base": 48.72471958402083, "excess": 5.552402086529744, "gap": 8.125789728967987, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 63.96752213183363, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 92.21621621621622, "hard": 60.36489462094999, "judge": 60.44825313117996, "standard": 62.32733361239012 } }, "open|noul": { "cc": 43.862663906142174, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 56.52173913043479, "judge": 10.000000000000009, "standard": 55.55555555555556 } }, "open|score": { "cc": 50.53265730753868, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 41.64121212121212, "hard": 47.127870370370395, "judge": 62.23203389830508, "standard": 44.23888888888891 } }, "sealed|choice": { "cc": 58.35475838263821, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 73.3944276106718, "hard": 49.60407239819003, "judge": 60.89385474860336, "standard": 64.52765118857003 } }, "sealed|noul": { "cc": 27.440215333904295, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 48.0, "hard": 2.9126213592232997, "judge": 36.8421052631579, "standard": 52.112676056338024 } }, "sealed|score": { "cc": 48.19050044206803, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 62.689858156028365, "hard": 43.85012254901962, "judge": 53.23419161676646, "standard": 42.05604060913702 } } } }, "jevbench_score": 24.772887446577812, "key": "decider-35b-a3b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 29, "ranked": true, "ranks": { "A": 29, "B": 27, "C": 31, "capability": 20 }, "scores": { "A": 24.772887446577812, "B": 24.135656537463312, "C": 17.20339406012348 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2493798840790987, "p50_s_raw": 0.04968994203954935, "p95_s_adjusted": 0.29907567272894087, "p95_s_raw": 0.07453783636447042 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.22742585943612, 84.9038621148463 ], "I": [ 43.19083908199794, 52.63193908662482 ], "capability": [ 61.40117742704215, 67.98807044411026 ], "composite": { "A": [ 18.69889165560131, 26.594050353427875 ], "B": [ 17.887370941224734, 26.251970770447716 ], "C": [ 12.985341427500913, 20.369267436681472 ] }, "seed": 27 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 83.61830413355554, "capability": 66.1715118587882, "intelligence": 48.72471958402083 }, "per_type": { "choice": { "accuracy": 0.692, "calibration": 78.68024999999997, "cc": 60.611198999649474, "cc_by_type": { "choice": 60.611198999649474 }, "cc_tiered": 61.16114025723592, "cc_tiered_open": 63.96752213183363, "cc_tiered_sealed": 58.35475838263821, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 60.611198999649474 }, "noul": { "accuracy": 0.6693333333333333, "calibration": 85.90816, "cc": 33.86666666666667, "cc_by_type": { "noul": 33.86666666666667 }, "cc_tiered": 35.65143962002323, "cc_tiered_open": 43.862663906142174, "cc_tiered_sealed": 27.440215333904295, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 33.86666666666667 }, "score": { "accuracy": 0.5893333333333334, "calibration": 86.26650240066667, "cc": 48.45208771929824, "cc_by_type": { "score": 48.45208771929824 }, "cc_tiered": 49.36157887480336, "cc_tiered_open": 50.53265730753868, "cc_tiered_sealed": 48.19050044206803, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 48.45208771929824 } }, "raw_on_A": { "calibration": 80.25053271222222, "capability": 65.45225190461375, "intelligence": 50.65397109700527 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 22.248611212569305, "Emphasis on Accuracy 60:20:20": 22.101309301010737, "Emphasis on Cost 20:20:60": 18.91495293933302, "Emphasis on Speed 20:60:20": 27.229064334478497, "Intelligence only": 21.8839776409655 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/Mapika/decider-35b-a3b", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "Qwen3.5-35B-A3B-Base with a trained decision readout, 34.7B parameters / 3B active", "adapter_id": "images/decider-35b-a3b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7386666666666667, "acc_among_decisive": 0.9061371841155235, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "oraculumai", "axes": { "calibration": 84.0077997110931, "cost": 64.32996474661243, "intelligence": 32.54303915979197, "speed": 90.7347510499745 }, "calibration": { "parts": { "choice": { "ece_hard": 0.07560407620955467, "ece_score": 84.87918475808907, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 84.87918475808907 }, "noul": { "abstention_rate": 0.37735849056603776, "brier": 0.1368158290573463, "ece": 0.05776783172439211, "n": 371, "score": 88.44643365512158 }, "score": { "interval90_coverage": 0.9051490514905149, "interval90_width": 0.4345076784101176, "n": 369, "nrps": 0.12172060479348325, "score": 78.69778072006868, "top_ece": 0.1521618904025716 } }, "score": 84.0077997110931 }, "capability": 58.27541943544253, "composite_ci95": { "A": [ 17.54626281581445, 31.019351182791876 ], "B": [ 14.871831849106172, 27.358180563302277 ], "C": [ 12.18490473320448, 21.54121609916103 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: frozen 25 Sep DeepInfra Qwen/Qwen3.5-4B market reference, already marked deprecated at that cutoff (replaced by Qwen/Qwen3.5-9B). $0.03/M input, zero generated output; 836,500 measured input tokens across 1,624 decisions. Frozen v1.5 base-model floor applied; no bookable Manchego tariff claimed. Re-score if the basis changes.", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.015452586206896551 }, "display": "Manchego v2.1", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 40.1322357986733, "I_sealed": 26.800244420595554, "base": 33.46624010963443, "excess": 10.758603735639502, "gap": 13.331991378077745, "penalty": 0.972413962643605, "per_type_split": { "open|choice": { "cc": 60.640330926520605, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 80.54054054054055, "hard": 45.26580685750237, "judge": 68.35860250494396, "standard": 69.8618668899121 } }, "open|noul": { "cc": 19.02691511387163, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 13.043478260869556, "judge": 0.0, "standard": 33.33333333333333 } }, "open|score": { "cc": 40.72946135562766, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 63.415593474232956, "hard": 33.07729129866341, "judge": 27.484035794573337, "standard": 64.558873751835 } }, "sealed|choice": { "cc": 56.6033475264085, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 65.41275589387332, "hard": 49.60407239819003, "judge": 56.238361266294234, "standard": 66.7446729892844 } }, "sealed|noul": { "cc": 2.9507869907230795, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 39.99999999999999, "hard": -30.097087378640776, "judge": 13.157894736842103, "standard": 35.2112676056338 } }, "sealed|score": { "cc": 20.84659874465509, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 52.898828557922364, "hard": 20.484461869906934, "judge": 15.149834013146535, "standard": 14.089904684780608 } } } }, "jevbench_score": 24.487333472500133, "key": "manchego-v21-v15", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 30, "ranked": true, "ranks": { "A": 30, "B": 30, "C": 32, "capability": 40 }, "scores": { "A": 24.487333472500133, "B": 21.196520189880484, "C": 17.005092689236207 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2574284589994932, "p50_s_raw": 0.05371422949974658, "p95_s_adjusted": 0.32799584188469455, "p95_s_raw": 0.08899792094234726 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 77.8981045039492, 85.26223280687248 ], "I": [ 28.50329098429543, 35.95139455461947 ], "capability": [ 54.23896847458738, 59.77267571842297 ], "composite": { "A": [ 17.54626281581445, 31.019351182791876 ], "B": [ 14.871831849106172, 27.358180563302277 ], "C": [ 12.18490473320448, 21.54121609916103 ] }, "seed": 65 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 84.0077997110931, "capability": 58.27541943544253, "intelligence": 32.54303915979197 }, "per_type": { "choice": { "accuracy": 0.6693333333333333, "calibration": 84.87918475808907, "cc": 57.712456068887754, "cc_by_type": { "choice": 57.712456068887754 }, "cc_tiered": 58.62183922646455, "cc_tiered_open": 60.640330926520605, "cc_tiered_sealed": 56.6033475264085, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 57.712456068887754 }, "noul": { "accuracy": 0.5573333333333333, "calibration": 88.44643365512158, "cc": 11.466666666666669, "cc_by_type": { "noul": 11.466666666666669 }, "cc_tiered": 10.988851052297356, "cc_tiered_open": 19.02691511387163, "cc_tiered_sealed": 2.9507869907230795, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 11.466666666666669 }, "score": { "accuracy": 0.49066666666666664, "calibration": 78.69778072006868, "cc": 26.44898115857113, "cc_by_type": { "score": 26.44898115857113 }, "cc_tiered": 30.788030050141376, "cc_tiered_open": 40.72946135562766, "cc_tiered_sealed": 20.84659874465509, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 26.44898115857113 } }, "raw_on_A": { "calibration": 84.55956989627305, "capability": 59.12164774190826, "intelligence": 33.68372558754347 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 22.181162996647437, "Emphasis on Accuracy 60:20:20": 17.83638056849814, "Emphasis on Cost 20:20:60": 23.964666214390906, "Emphasis on Speed 20:60:20": 26.697557732929795, "Intelligence only": 13.785874409327832 }, "class": "system-one-open", "open": "yes", "licence": "Apache-2.0 (code and weights, author-declared)", "repo": "https://github.com/nschlaepfer/manchego-serve", "gpu": "RTX6000", "endpoint_condition": "evaluator-owned offline RTX6000 GPU", "underlying": "Author server manchego-serve v0.1.1 @ 0ee62188af474e3c1d11d47a9c4c8b3a08766db3; oraculumai/Manchego @ 77403228b7dfdf823af99a5f562bdcf80b708d4c. Evaluator-owned RTX6000, offline/read-only container, safetensors with verified weight and support-file checksums, bfloat16, sequential execution, temperature 1.0 and original option order. Stock TypeSafe adapter; all three types native. The author disclosed using public aggregate/family results during development, a prior Jev-assisted phrase filter on two template pools, and one Jev-labelled development group for checkpoint selection. These disclosures do not establish item-text leakage; this is not a claim of fully independent development. Cost is an estimate using measured token usage and the frozen Qwen3.5-4B reference, not a bookable Manchego tariff.", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6226415094339622, "acc_among_decisive": 0.9047619047619048, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "unknown", "axes": { "calibration": 88.42488519968094, "cost": 63.07359395281681, "intelligence": 32.40310972936693, "speed": 93.92936299294371 }, "calibration": { "parts": { "choice": { "ece_hard": 0.0672265434886423, "ece_score": 86.55469130227154, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 86.55469130227154 }, "noul": { "abstention_rate": 0.41239892183288407, "brier": 0.12038668872149434, "ece": 0.03739565815349834, "n": 371, "score": 92.52086836930033 }, "score": { "interval90_coverage": 0.975609756097561, "interval90_width": 0.5718157181571816, "n": 369, "nrps": 0.10515132123479526, "score": 86.19909592747098, "top_ece": 0.08543338010789257 } }, "score": 88.42488519968094 }, "capability": 60.41399746452394, "composite_ci95": { "A": [ 15.392618056749317, 34.34955627190713 ], "B": [ 12.888673039303178, 30.38967783428378 ], "C": [ 10.689318094964802, 23.853858522157736 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01701689039408867 }, "display": "Decision 4B v1.2 (FlyMyJev, Qwen3.5-4B + LoRA)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 32.99062805982875, "I_sealed": 31.81559139890512, "base": 32.40310972936693, "excess": -1.398350981514613, "gap": 1.175036660923631, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 61.39828743553336, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 80.54054054054055, "hard": 50.92796476879523, "judge": 68.35860250494396, "standard": 62.32733361239012 } }, "open|noul": { "cc": 6.031055900621121, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 28.57142857142858, "hard": 30.434782608695656, "judge": -30.000000000000004, "standard": 0.0 } }, "open|score": { "cc": 31.542540843331764, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 49.77190646723929, "hard": 38.72644314542413, "judge": 25.494038632486827, "standard": 17.13280674346067 } }, "sealed|choice": { "cc": 58.83327326304771, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 68.07331313280615, "hard": 56.730769230769226, "judge": 56.238361266294234, "standard": 62.31062938785566 } }, "sealed|noul": { "cc": 8.040955184350867, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 28.000000000000004, "hard": -8.737864077669899, "judge": 13.157894736842103, "standard": 23.943661971830977 } }, "sealed|score": { "cc": 28.5725457493168, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 58.99125190199095, "hard": 23.870346180703184, "judge": 26.811123175914563, "standard": 25.409725670310312 } } } }, "jevbench_score": 24.463556699266523, "key": "decision-4b-v12", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 31, "ranked": true, "ranks": { "A": 31, "B": 31, "C": 33, "capability": 36 }, "scores": { "A": 24.463556699266523, "B": 21.09790572169507, "C": 16.988581041157314 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.18829862689599394, "p50_s_raw": 0.019149313447996974, "p95_s_adjusted": 0.2148901690961793, "p95_s_raw": 0.032445084548089656 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 81.81349929936302, 88.39912781374252 ], "I": [ 26.905961082898493, 37.32153047837871 ], "capability": [ 55.53187740606071, 61.760259132085466 ], "composite": { "A": [ 15.392618056749317, 34.34955627190713 ], "B": [ 12.888673039303178, 30.38967783428378 ], "C": [ 10.689318094964802, 23.853858522157736 ] }, "seed": 31 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 88.42488519968094, "capability": 60.41399746452394, "intelligence": 32.40310972936693 }, "per_type": { "choice": { "accuracy": 0.688, "calibration": 86.55469130227154, "cc": 60.099656129515054, "cc_by_type": { "choice": 60.099656129515054 }, "cc_tiered": 60.11578034929053, "cc_tiered_open": 61.39828743553336, "cc_tiered_sealed": 58.83327326304771, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 60.099656129515054 }, "noul": { "accuracy": 0.5493333333333333, "calibration": 92.52086836930033, "cc": 9.866666666666667, "cc_by_type": { "noul": 9.866666666666667 }, "cc_tiered": 7.036005542485994, "cc_tiered_open": 6.031055900621121, "cc_tiered_sealed": 8.040955184350867, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 9.866666666666667 }, "score": { "accuracy": 0.5173333333333333, "calibration": 86.19909592747098, "cc": 30.992722913904714, "cc_by_type": { "score": 30.992722913904714 }, "cc_tiered": 30.057543296324283, "cc_tiered_open": 31.542540843331764, "cc_tiered_sealed": 28.5725457493168, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 30.992722913904714 } }, "raw_on_A": { "calibration": 81.8292718059759, "capability": 58.67224928043624, "intelligence": 35.515226754896574 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 21.964945240595316, "Emphasis on Accuracy 60:20:20": 17.633890573157025, "Emphasis on Cost 20:20:60": 23.575828038111915, "Emphasis on Speed 20:60:20": 26.69802362301605, "Intelligence only": 13.608807339398107 }, "class": "unclassified", "open": "unknown", "licence": "not recorded", "repo": "https://benchmarkheaven.com/jev-models/decision-4b-v12", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": null, "adapter_id": "images/decision-4b-v12/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5876010781671159, "acc_among_decisive": 0.944954128440367, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Alibaba Qwen / neutral reproduction", "axes": { "calibration": 35.18175191334879, "cost": 63.36068112101963, "intelligence": 33.95117742100547, "speed": 91.16296741568695 }, "calibration": { "parts": { "choice": { "ece_hard": 0.44119675391398117, "ece_score": 11.760649217203767, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 11.760649217203767 }, "noul": { "abstention_rate": 0.07547169811320753, "brier": 0.26616192586960774, "ece": 0.254956633051564, "n": 371, "score": 49.008673389687196 }, "score": { "interval90_coverage": 0.5501355013550135, "interval90_width": 0.07452574525745256, "n": 369, "nrps": 0.1781187945424487, "score": 44.77593313315542, "top_ece": 0.46318127139722143 } }, "score": 35.18175191334879 }, "capability": 34.56646466717713, "composite_ci95": { "A": [ 13.8369005003162, 28.950164844761904 ], "B": [ 12.606782226433888, 27.192389151058098 ], "C": [ 9.608958680775137, 20.10428114219577 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01664602832512315 }, "display": "Raw Qwen3 4B Instruct 2507 direct logits", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 39.33458111901565, "I_sealed": 28.646049395387923, "base": 33.990315257201786, "excess": 8.115144081189486, "gap": 10.68853172362773, "penalty": 0.9988485591881051, "per_type_split": { "open|choice": { "cc": 53.102399069057086, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 68.86486486486487, "hard": 39.60364894620951, "judge": 44.62755438365195, "standard": 84.93093344495604 } }, "open|noul": { "cc": 40.36853002070393, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 47.82608695652173, "judge": 19.999999999999996, "standard": 33.33333333333333 } }, "open|score": { "cc": 24.532814267285957, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 44.98343409680498, "hard": 0.7007591330399521, "judge": 15.311071908696839, "standard": 75.80422815890212 } }, "sealed|choice": { "cc": 41.64735120428594, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 53.44024831867564, "hard": 30.2601809954751, "judge": 41.34078212290503, "standard": 58.9850966867841 } }, "sealed|noul": { "cc": 25.067667527906327, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 76.0, "hard": 0.9708737864077666, "judge": 31.578947368421062, "standard": 38.0281690140845 } }, "sealed|score": { "cc": 19.223129453971513, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 48.978480535664694, "hard": 7.881002538538395, "judge": 13.283448182092084, "standard": 35.9392296518103 } } } }, "jevbench_score": 21.792436038282993, "key": "raw-qwen3-4b-instruct-2507", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 32, "ranked": true, "ranks": { "A": 32, "B": 33, "C": 36, "capability": 77 }, "scores": { "A": 21.792436038282993, "B": 20.20761454160472, "C": 15.133636137696522 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.24207517188042402, "p50_s_raw": 0.04603758594021201, "p95_s_adjusted": 0.31604797318577765, "p95_s_raw": 0.08302398659288883 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 30.079848447424265, 39.87330204447962 ], "I": [ 28.52380607287802, 37.88233031513499 ], "capability": [ 29.696940965810747, 38.2358105496228 ], "composite": { "A": [ 13.8369005003162, 28.950164844761904 ], "B": [ 12.606782226433888, 27.192389151058098 ], "C": [ 9.608958680775137, 20.10428114219577 ] }, "seed": 96 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 35.18175191334879, "capability": 34.56646466717713, "intelligence": 33.95117742100547 }, "per_type": { "choice": { "accuracy": 0.5626666666666666, "calibration": 11.760649217203767, "cc": 44.07131286530316, "cc_by_type": { "choice": 44.07131286530316 }, "cc_tiered": 47.37487513667151, "cc_tiered_open": 53.102399069057086, "cc_tiered_sealed": 41.64735120428594, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 44.07131286530316 }, "noul": { "accuracy": 0.664, "calibration": 49.008673389687196, "cc": 32.800000000000004, "cc_by_type": { "noul": 32.800000000000004 }, "cc_tiered": 32.718098774305126, "cc_tiered_open": 40.36853002070393, "cc_tiered_sealed": 25.067667527906327, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 32.800000000000004 }, "score": { "accuracy": 0.456, "calibration": 44.77593313315542, "cc": 20.99121192415334, "cc_by_type": { "score": 20.99121192415334 }, "cc_tiered": 21.877971860628733, "cc_tiered_open": 24.532814267285957, "cc_tiered_sealed": 19.223129453971513, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 20.99121192415334 } }, "raw_on_A": { "calibration": 37.75992930947443, "capability": 38.17464116904609, "intelligence": 38.58935302861775 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 24.609772187762267, "Emphasis on Accuracy 60:20:20": 20.02676319678133, "Emphasis on Cost 20:20:60": 26.265558574780524, "Emphasis on Speed 20:60:20": 29.50118725011904, "Intelligence only": 15.653970524552685 }, "class": "raw-logit-control", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen3-4B-Instruct-2507 BF16", "adapter_id": "images/raw-qwen3-4b-instruct-2507/spec.json:raw_adapter.RawControlAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.9245283018867925, "acc_among_decisive": 0.7259475218658892, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Bespoke Labs", "axes": { "calibration": 69.88762704651819, "cost": 36.75037423027614, "intelligence": 43.14237010986392, "speed": 86.11327010751721 }, "calibration": { "parts": { "choice": { "ece_hard": 0.178988177870932, "ece_score": 64.2023644258136, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 64.2023644258136 }, "noul": { "abstention_rate": 0.2264150943396226, "brier": 0.14509070919724038, "ece": 0.1305339256295796, "n": 371, "score": 73.89321487408408 }, "score": { "interval90_coverage": 0.8401084010840109, "interval90_width": 0.26874435411020764, "n": 369, "nrps": 0.11567566172039558, "score": 71.56730183965689, "top_ece": 0.22648915074323328 } }, "score": 69.88762704651819 }, "capability": 56.514998578191054, "composite_ci95": { "A": [ 15.043066969876142, 26.052957246349575 ], "B": [ 14.110886674014932, 25.283661649629934 ], "C": [ 10.446574284636212, 18.092331421076096 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.1283278325123153 }, "display": "Bespoke Nimble 9B (Bespoke Labs)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 46.96255933707536, "I_sealed": 39.32218088265248, "base": 43.14237010986392, "excess": 5.0669908119846365, "gap": 7.64037845442288, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 59.86495466898259, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675674, "hard": 49.040578798364265, "judge": 68.35860250494396, "standard": 62.32733361239012 } }, "open|noul": { "cc": 38.9399585921325, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 47.82608695652173, "judge": 19.999999999999996, "standard": 33.33333333333333 } }, "open|score": { "cc": 42.08276475011101, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 47.18206312365201, "hard": 44.02495901721014, "judge": 36.708929948504554, "standard": 43.70947923155191 } }, "sealed|choice": { "cc": 57.35467556665829, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 60.0916414160077, "hard": 46.5497737556561, "judge": 60.89385474860336, "standard": 72.28722749107035 } }, "sealed|noul": { "cc": 27.490153799650226, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 52.0, "hard": 4.854368932038833, "judge": 36.8421052631579, "standard": 46.47887323943662 } }, "sealed|score": { "cc": 33.12171328164895, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 49.98365384887967, "hard": 27.146627400052527, "judge": 32.82017491999325, "standard": 37.093222303709986 } } } }, "jevbench_score": 21.082616704957896, "key": "nimble-9b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 33, "ranked": true, "ranks": { "A": 33, "B": 32, "C": 37, "capability": 45 }, "scores": { "A": 21.082616704957896, "B": 20.213517776427228, "C": 14.64070604510965 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.43987150778993966, "p50_s_raw": 0.14493575389496982, "p95_s_adjusted": 0.5563488021492958, "p95_s_raw": 0.2031744010746479 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 65.1956012319158, 73.91457472170808 ], "I": [ 37.37380617272022, 47.11791102705384 ], "capability": [ 52.152502588110366, 59.801663230716926 ], "composite": { "A": [ 15.043066969876142, 26.052957246349575 ], "B": [ 14.110886674014932, 25.283661649629934 ], "C": [ 10.446574284636212, 18.092331421076096 ] }, "seed": 71 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 69.88762704651819, "capability": 56.514998578191054, "intelligence": 43.14237010986392 }, "per_type": { "choice": { "accuracy": 0.668, "calibration": 64.2023644258136, "cc": 57.541941778842954, "cc_by_type": { "choice": 57.541941778842954 }, "cc_tiered": 58.60981511782044, "cc_tiered_open": 59.86495466898259, "cc_tiered_sealed": 57.35467556665829, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 57.541941778842954 }, "noul": { "accuracy": 0.664, "calibration": 73.89321487408408, "cc": 32.800000000000004, "cc_by_type": { "noul": 32.800000000000004 }, "cc_tiered": 33.215056195891364, "cc_tiered_open": 38.9399585921325, "cc_tiered_sealed": 27.490153799650226, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 32.800000000000004 }, "score": { "accuracy": 0.536, "calibration": 71.56730183965689, "cc": 35.587315474154515, "cc_by_type": { "score": 35.587315474154515 }, "cc_tiered": 37.60223901587998, "cc_tiered_open": 42.08276475011101, "cc_tiered_sealed": 33.12171328164895, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 35.587315474154515 } }, "raw_on_A": { "calibration": 72.02241547589321, "capability": 59.4874434751425, "intelligence": 46.95247147439178 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 19.46099526058868, "Emphasis on Accuracy 60:20:20": 18.55883910946502, "Emphasis on Cost 20:20:60": 17.273520313490142, "Emphasis on Speed 20:60:20": 23.596180145578664, "Intelligence only": 17.35223877358268 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 (weights); repository without a licence file as of 19 Sep", "repo": "https://github.com/bespokelabsai/nimble", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "bespokelabs/Bespoke-Nimble-9B (LoRA, adapter unchanged since 93ec5d6), merged into Qwen/Qwen3.5-9B@c202236 with the author's PEFT safe-merge", "adapter_id": "images/nimble-9b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7735849056603774, "acc_among_decisive": 0.867595818815331, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "kshetrajna12", "axes": { "calibration": 86.97925577932901, "cost": 63.142712976263866, "intelligence": 30.935558296483705, "speed": 68.7900017304639 }, "calibration": { "parts": { "choice": { "ece_hard": 0.068294434375, "ece_score": 86.341113125, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 86.341113125 }, "noul": { "abstention_rate": 0.41866666666666663, "brier": 0.14083422852570657, "ece": 0.05314781866666669, "n": 375, "score": 89.37043626666666 }, "score": { "interval90_coverage": 0.9866666666666667, "interval90_width": 0.7284444444444443, "n": 375, "nrps": 0.1108956250735923, "score": 85.22621794632039, "top_ece": 0.09229000799999995 } }, "score": 86.97925577932901 }, "capability": 58.95740703790636, "composite_ci95": { "A": [ 14.765436275328822, 29.209404384423753 ], "B": [ 12.581433816044255, 26.114502689493058 ], "C": [ 10.253775191200571, 20.28430860029427 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.016926853448275863 }, "display": "reflex 4B (kshetrajna12)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 33.322266958336286, "I_sealed": 28.548849634631125, "base": 30.935558296483705, "excess": 2.200029681266917, "gap": 4.773417323705161, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 55.076336216021176, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864865, "hard": 50.92796476879523, "judge": 48.58272907053395, "standard": 62.32733361239012 } }, "open|noul": { "cc": 10.45962732919255, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 42.85714285714286, "hard": 30.434782608695656, "judge": -19.999999999999996, "standard": 0.0 } }, "open|score": { "cc": 34.430837329795125, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 52.20111818181819, "hard": 24.787801851851842, "judge": 33.86929491525422, "standard": 45.67408148148149 } }, "sealed|choice": { "cc": 53.57907452177943, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 65.41275589387332, "hard": 50.113122171945705, "judge": 50.65176908752329, "standard": 58.9850966867841 } }, "sealed|noul": { "cc": 1.7255356358899414, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 39.99999999999999, "hard": -14.563106796116509, "judge": 5.263157894736836, "standard": 9.859154929577475 } }, "sealed|score": { "cc": 30.341938746224002, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 54.91348368794326, "hard": 26.727804411764712, "judge": 23.787971856287427, "standard": 35.11538527918782 } } } }, "jevbench_score": 20.637311182702398, "key": "reflex-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 34, "ranked": true, "ranks": { "A": 34, "B": 37, "C": 38, "capability": 38 }, "scores": { "A": 20.637311182702398, "B": 17.968342404009285, "C": 14.33146609909889 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 2.865421538101509, "p50_s_raw": 1.3577107690507546, "p95_s_adjusted": 4.611171829840169, "p95_s_raw": 2.230585914920084 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 80.53682470259905, 87.39459423959244 ], "I": [ 27.05693352837418, 35.79974193094454 ], "capability": [ 54.86352340583442, 60.49053268982462 ], "composite": { "A": [ 14.765436275328822, 29.209404384423753 ], "B": [ 12.581433816044255, 26.114502689493058 ], "C": [ 10.253775191200571, 20.28430860029427 ] }, "seed": 98 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 86.97925577932901, "capability": 58.95740703790636, "intelligence": 30.935558296483705 }, "per_type": { "choice": { "accuracy": 0.648, "calibration": 86.341113125, "cc": 54.98422742817084, "cc_by_type": { "choice": 54.98422742817084 }, "cc_tiered": 54.327705368900304, "cc_tiered_open": 55.076336216021176, "cc_tiered_sealed": 53.57907452177943, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 54.98422742817084 }, "noul": { "accuracy": 0.5333333333333333, "calibration": 89.37043626666666, "cc": 6.666666666666665, "cc_by_type": { "noul": 6.666666666666665 }, "cc_tiered": 6.092581482541245, "cc_tiered_open": 10.45962732919255, "cc_tiered_sealed": 1.7255356358899414, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 6.666666666666665 }, "score": { "accuracy": 0.5173333333333333, "calibration": 85.22621794632039, "cc": 32.60462780701755, "cc_by_type": { "score": 32.60462780701755 }, "cc_tiered": 32.386388038009564, "cc_tiered_open": 34.430837329795125, "cc_tiered_sealed": 30.341938746224002, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 32.60462780701755 } }, "raw_on_A": { "calibration": 81.52279417962764, "capability": 58.44282614389385, "intelligence": 35.36285810816006 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 18.316133226478374, "Emphasis on Accuracy 60:20:20": 15.029593546249647, "Emphasis on Cost 20:20:60": 20.28126457022741, "Emphasis on Speed 20:60:20": 20.855909642703498, "Intelligence only": 11.842240202134587 }, "class": "jev-rebuild", "open": "yes", "licence": "MIT (code, adapter); Apache-2.0 (base)", "repo": "https://github.com/kshetrajna12/reflex", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "Qwen/Qwen3.5-4B + kshetrajna12/reflex-qwen3.5-4b-lora", "adapter_id": "images/reflex-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5813333333333334, "acc_among_decisive": 0.9174311926605505, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Mikhail Gribov", "axes": { "calibration": 83.29951051808085, "cost": 63.95900966831593, "intelligence": 29.78900656204624, "speed": 92.76354061232337 }, "calibration": { "parts": { "choice": { "ece_hard": 0.14894691278376873, "ece_score": 70.21061744324626, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 70.21061744324626 }, "noul": { "abstention_rate": 0.42666666666666664, "brier": 0.1443460310902931, "ece": 0.03743872703084413, "n": 375, "score": 92.51225459383117 }, "score": { "interval90_coverage": 0.9973333333333333, "interval90_width": 0.8611111111111112, "n": 375, "nrps": 0.11438335298438909, "score": 87.17565951716512, "top_ece": 0.07105172833615442 } }, "score": 83.29951051808085 }, "capability": 56.544258540063545, "composite_ci95": { "A": [ 12.880576558614674, 26.104438970675695 ], "B": [ 10.666557109719218, 22.72472546215454 ], "C": [ 8.944844832371304, 18.12808261852479 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01589887315270936 }, "display": "typecastlm (Mikhail Gribov, Qwen3.5-4B computed head)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 34.211044348094305, "I_sealed": 25.366968775998174, "base": 29.78900656204624, "excess": 6.270687929657887, "gap": 8.84407557209613, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 53.948175356316256, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 61.08108108108109, "hard": 45.26580685750237, "judge": 52.537903757415954, "standard": 69.8618668899121 } }, "open|noul": { "cc": 19.30296756383713, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 14.28571428571428, "hard": 39.13043478260869, "judge": 0.0, "standard": 11.111111111111116 } }, "open|score": { "cc": 29.381990124129523, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 59.87815307647144, "hard": 32.63833455597018, "judge": 28.21824515037382, "standard": 9.366837244910798 } }, "sealed|choice": { "cc": 46.35542091610937, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 61.4219200354741, "hard": 37.89592760180995, "judge": 51.58286778398511, "standard": 47.89998768321223 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 19.999999999999996, "hard": -22.330097087378643, "judge": 2.6315789473684292, "standard": 15.492957746478876 } }, "sealed|score": { "cc": 29.745485411885156, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 52.652319160951166, "hard": 25.940906558427802, "judge": 26.848256959038974, "standard": 30.247068923536112 } } } }, "jevbench_score": 19.72256430004839, "key": "typecastlm", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 35, "ranked": true, "ranks": { "A": 35, "B": 40, "C": 39, "capability": 44 }, "scores": { "A": 19.72256430004839, "B": 16.813084809424783, "C": 13.69622520836694 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.20903298105113208, "p50_s_raw": 0.02951649052556604, "p95_s_adjusted": 0.25318100801086985, "p95_s_raw": 0.051590504005434926 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 77.58498026450093, 85.01640087455884 ], "I": [ 25.093945135273305, 33.49496683838513 ], "capability": [ 52.490530265893305, 58.297640238106396 ], "composite": { "A": [ 12.880576558614674, 26.104438970675695 ], "B": [ 10.666557109719218, 22.72472546215454 ], "C": [ 8.944844832371304, 18.12808261852479 ] }, "seed": 107 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 83.29951051808085, "capability": 56.544258540063545, "intelligence": 29.78900656204624 }, "per_type": { "choice": { "accuracy": 0.5933333333333334, "calibration": 70.21061744324626, "cc": 47.99314153633374, "cc_by_type": { "choice": 47.99314153633374 }, "cc_tiered": 50.15179813621281, "cc_tiered_open": 53.948175356316256, "cc_tiered_sealed": 46.35542091610937, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 47.99314153633374 }, "noul": { "accuracy": 0.5173333333333333, "calibration": 92.51225459383117, "cc": 3.4666666666666623, "cc_by_type": { "noul": 3.4666666666666623 }, "cc_tiered": 9.651483781918564, "cc_tiered_open": 19.30296756383713, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 3.4666666666666623 }, "score": { "accuracy": 0.48533333333333334, "calibration": 87.17565951716512, "cc": 31.463946158492806, "cc_by_type": { "score": 31.463946158492806 }, "cc_tiered": 29.56373776800734, "cc_tiered_open": 29.381990124129523, "cc_tiered_sealed": 29.745485411885156, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 31.463946158492806 } }, "raw_on_A": { "calibration": 79.3986887571586, "capability": 55.69880635159235, "intelligence": 31.998923946026117 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 17.752271764169233, "Emphasis on Accuracy 60:20:20": 13.960999831173599, "Emphasis on Cost 20:20:60": 19.448558330257274, "Emphasis on Speed 20:60:20": 21.764369978155518, "Intelligence only": 10.57372598609905 }, "class": "system-one-open", "open": "yes", "licence": "Apache-2.0 (package and weights)", "repo": "https://github.com/mihail-gribov/typecastlm", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": null, "adapter_id": "images/typecastlm/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5733333333333334, "acc_among_decisive": 0.9023255813953488, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "unknown", "axes": { "calibration": 85.23934158755043, "cost": 63.07359395281681, "intelligence": 29.134168280172865, "speed": 93.95641941218838 }, "calibration": { "parts": { "choice": { "ece_hard": 0.10557670546665254, "ece_score": 78.88465890666949, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 78.88465890666949 }, "noul": { "abstention_rate": 0.4339622641509434, "brier": 0.11841873108613687, "ece": 0.04782018815907789, "n": 371, "score": 90.43596236818442 }, "score": { "interval90_coverage": 0.9701897018970189, "interval90_width": 0.5756549232158987, "n": 369, "nrps": 0.11671345785768895, "score": 86.39740348779742, "top_ece": 0.07766923619318136 } }, "score": 85.23934158755043 }, "capability": 57.18675493386165, "composite_ci95": { "A": [ 12.444491289404162, 27.25430281789905 ], "B": [ 10.295694887525865, 23.807726232797425 ], "C": [ 8.642007839864, 18.926599179096563 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01701689039408867 }, "display": "Decision 4B v1.1 (FlyMyJev, Qwen3.5-4B + LoRA)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 30.696800084915584, "I_sealed": 27.571536475430143, "base": 29.134168280172865, "excess": 0.5518759670471964, "gap": 3.1252636094854402, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 60.19131316010312, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675674, "hard": 56.59012268008808, "judge": 64.40342781806196, "standard": 54.79280033486815 } }, "open|noul": { "cc": 1.1380262249827418, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 13.043478260869556, "judge": -30.000000000000004, "standard": -11.111111111111116 } }, "open|score": { "cc": 30.761060869660888, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 54.918373499249086, "hard": 31.938975913174172, "judge": 30.29541690533948, "standard": 17.025040414322312 } }, "sealed|choice": { "cc": 57.815173715536396, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 68.07331313280615, "hard": 54.185520361990946, "judge": 56.238361266294234, "standard": 62.31062938785566 } }, "sealed|noul": { "cc": 2.4545445385650666, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 28.000000000000004, "hard": -16.504854368932044, "judge": 10.526315789473696, "standard": 15.492957746478876 } }, "sealed|score": { "cc": 22.444891172188967, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 55.42678299206407, "hard": 20.963579856299262, "judge": 15.25087787708388, "standard": 19.707587836688447 } } } }, "jevbench_score": 18.718356274892123, "key": "decision-4b-v11", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 36, "ranked": true, "ranks": { "A": 36, "B": 42, "C": 41, "capability": 42 }, "scores": { "A": 18.718356274892123, "B": 15.883627667705545, "C": 12.998858524230638 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.188053295109421, "p50_s_raw": 0.019026647554710507, "p95_s_adjusted": 0.21383417290635406, "p95_s_raw": 0.031917086453177035 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 79.26750110051037, 86.2633177195094 ], "I": [ 24.784375611108008, 34.07951912545236 ], "capability": [ 52.99615856993229, 59.129104174768116 ], "composite": { "A": [ 12.444491289404162, 27.25430281789905 ], "B": [ 10.295694887525865, 23.807726232797425 ], "C": [ 8.642007839864, 18.926599179096563 ] }, "seed": 30 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 85.23934158755043, "capability": 57.18675493386165, "intelligence": 29.134168280172865 }, "per_type": { "choice": { "accuracy": 0.68, "calibration": 78.88465890666949, "cc": 59.07657038924623, "cc_by_type": { "choice": 59.07657038924623 }, "cc_tiered": 59.00324343781976, "cc_tiered_open": 60.19131316010312, "cc_tiered_sealed": 57.815173715536396, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 59.07657038924623 }, "noul": { "accuracy": 0.528, "calibration": 90.43596236818442, "cc": 5.600000000000005, "cc_by_type": { "noul": 5.600000000000005 }, "cc_tiered": 1.7962853817739042, "cc_tiered_open": 1.1380262249827418, "cc_tiered_sealed": 2.4545445385650666, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 5.600000000000005 }, "score": { "accuracy": 0.47733333333333333, "calibration": 86.39740348779742, "cc": 26.584587548466242, "cc_by_type": { "score": 26.584587548466242 }, "cc_tiered": 26.602976020924928, "cc_tiered_open": 30.761060869660888, "cc_tiered_sealed": 22.444891172188967, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 26.584587548466242 } }, "raw_on_A": { "calibration": 85.6538943241602, "capability": 60.62425637183274, "intelligence": 35.59461841950529 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 16.746653513023, "Emphasis on Accuracy 60:20:20": 13.11195414676558, "Emphasis on Cost 20:20:60": 18.346342760998827, "Emphasis on Speed 20:60:20": 20.675164334326784, "Intelligence only": 9.891630033656257 }, "class": "unclassified", "open": "unknown", "licence": "not recorded", "repo": "https://benchmarkheaven.com/jev-models/decision-4b-v11", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": null, "adapter_id": "images/decision-4b-v11/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5660377358490566, "acc_among_decisive": 0.9428571428571428, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "unknown", "axes": { "calibration": 87.61544276592178, "cost": 29.365937020749882, "intelligence": 55.62422706919757, "speed": 88.60757446254297 }, "calibration": { "parts": { "choice": { "ece_hard": 0.0794047531024999, "ece_score": 84.11904937950003, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 84.11904937950003 }, "noul": { "abstention_rate": 0.2964959568733153, "brier": 0.06886464230605442, "ece": 0.04566629583733653, "n": 371, "score": 90.8667408325327 }, "score": { "interval90_coverage": 1.0, "interval90_width": 0.41666666666666674, "n": 369, "nrps": 0.0623820892760947, "score": 87.86053808573263, "top_ece": 0.09020357450462646 } }, "score": 87.61544276592178 }, "capability": 71.61983491755967, "composite_ci95": { "A": [ 17.95132364974654, 18.715305818590913 ], "B": [ 17.85214158412037, 19.07667324580802 ], "C": [ 12.749857516255931, 18.58395156303398 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.22618758620689655 }, "display": "AutoJev-27B (denis-pplx, Qwen3.8-27B)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 56.79842954079213, "I_sealed": 54.45002459760302, "base": 55.62422706919757, "excess": -0.2249826992491344, "gap": 2.3484049431891094, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 79.34292176773366, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 75.46398238439761, "judge": 80.22412656558997, "standard": 77.39640016743407 } }, "open|noul": { "cc": 34.36853002070393, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 47.82608695652173, "judge": 0.0, "standard": 33.33333333333333 } }, "open|score": { "cc": 56.68383683393881, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 65.12454762185703, "hard": 56.31402454129169, "judge": 64.63259495424096, "standard": 41.2799688448207 } }, "sealed|choice": { "cc": 75.98476401702004, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 81.37609932747026, "hard": 68.43891402714931, "judge": 78.58472998137803, "standard": 84.48084739499939 } }, "sealed|noul": { "cc": 33.77589728457613, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 68.0, "hard": 12.621359223300965, "judge": 42.10526315789473, "standard": 46.47887323943662 } }, "sealed|score": { "cc": 53.589412491212904, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 68.78624407613681, "hard": 41.90521556676637, "judge": 62.130233048417736, "standard": 56.54815971183675 } } } }, "jevbench_score": 18.463427917707932, "key": "autojev-27b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 37, "ranked": true, "ranks": { "A": 37, "B": 35, "C": 35, "capability": 11 }, "scores": { "A": 18.463427917707932, "B": 18.603781697586662, "C": 15.868570767928437 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3447543190326542, "p50_s_raw": 0.09737715951632708, "p95_s_adjusted": 0.3996987941907718, "p95_s_raw": 0.12484939709538592 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 82.93064258139867, 88.05572734371755 ], "I": [ 50.54754866954701, 59.786942555692185 ], "capability": [ 67.70288142885737, 72.92248557654668 ], "composite": { "A": [ 17.95132364974654, 18.715305818590913 ], "B": [ 17.85214158412037, 19.07667324580802 ], "C": [ 12.749857516255931, 18.58395156303398 ] }, "seed": 16 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 87.61544276592178, "capability": 71.61983491755967, "intelligence": 55.62422706919757 }, "per_type": { "choice": { "accuracy": 0.82, "calibration": 84.11904937950003, "cc": 76.98057084395099, "cc_by_type": { "choice": 76.98057084395099 }, "cc_tiered": 77.66384289237685, "cc_tiered_open": 79.34292176773366, "cc_tiered_sealed": 75.98476401702004, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 76.98057084395099 }, "noul": { "accuracy": 0.688, "calibration": 90.8667408325327, "cc": 37.59999999999999, "cc_by_type": { "noul": 37.59999999999999 }, "cc_tiered": 34.07221365264003, "cc_tiered_open": 34.36853002070393, "cc_tiered_sealed": 33.77589728457613, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 37.59999999999999 }, "score": { "accuracy": 0.6506666666666666, "calibration": 87.86053808573263, "cc": 53.92178222051296, "cc_by_type": { "score": 53.92178222051296 }, "cc_tiered": 55.136624662575855, "cc_tiered_open": 56.68383683393881, "cc_tiered_sealed": 53.589412491212904, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 53.92178222051296 } }, "raw_on_A": { "calibration": 85.84423247942522, "capability": 73.42644032760776, "intelligence": 61.00864817579031 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 16.343749600153146, "Emphasis on Accuracy 60:20:20": 17.37362601712716, "Emphasis on Cost 20:20:60": 13.123431991427351, "Emphasis on Speed 20:60:20": 20.08099797467756, "Intelligence only": 19.187204603319074 }, "class": "unclassified", "open": "unknown", "licence": "not recorded", "repo": "https://huggingface.co/denis-pplx/autojev-27b", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": null, "adapter_id": "images/autojev-27b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7035040431266847, "acc_among_decisive": 0.9885057471264368, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "unknown", "axes": { "calibration": 89.97714437568723, "cost": 28.689752882469676, "intelligence": 64.40270761245412, "speed": 88.08630815059152 }, "calibration": { "parts": { "choice": { "ece_hard": 0.03950772533975903, "ece_score": 92.09845493204818, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 92.09845493204818 }, "noul": { "abstention_rate": 0.19137466307277629, "brier": 0.05632135167655414, "ece": 0.06181891259769947, "n": 371, "score": 87.6362174804601 }, "score": { "interval90_coverage": 0.964769647696477, "interval90_width": 0.3172990063233966, "n": 369, "nrps": 0.05868305372968562, "score": 90.19676071455342, "top_ece": 0.06869086598962304 } }, "score": 89.97714437568723 }, "capability": 77.18992599407068, "composite_ci95": { "A": [ 17.720899810915583, 18.261473654252256 ], "B": [ 18.087561450267057, 18.993796215955186 ], "C": [ 17.656534367876468, 18.261473654252256 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.23823646551724137 }, "display": "Eikos-27B (caiovicentino1, Qwen3.8-27B)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 65.62981582129679, "I_sealed": 63.17559940361144, "base": 64.40270761245412, "excess": -0.1191712247528991, "gap": 2.4542164176853447, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 80.41823040534584, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 77.35136835482857, "judge": 76.26895187870797, "standard": 84.93093344495604 } }, "open|noul": { "cc": 58.24775707384403, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 73.91304347826086, "judge": 30.000000000000004, "standard": 55.55555555555556 } }, "open|score": { "cc": 58.2234599847005, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 61.643002416487, "hard": 61.32359173476558, "judge": 58.76732285181849, "standard": 49.49763096800006 } }, "sealed|choice": { "cc": 78.56119802347493, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 76.05498484960461, "hard": 74.5475113122172, "judge": 78.58472998137803, "standard": 87.80638009607095 } }, "sealed|noul": { "cc": 53.35348874031106, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 76.0, "hard": 37.86407766990292, "judge": 57.89473684210527, "standard": 66.19718309859155 } }, "sealed|score": { "cc": 57.61211144704836, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 67.33566285144035, "hard": 49.71122434442663, "judge": 64.06787863213144, "standard": 58.86845917247121 } } } }, "jevbench_score": 18.077889022432696, "key": "eikos-27b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 38, "ranked": true, "ranks": { "A": 38, "B": 34, "C": 28, "capability": 4 }, "scores": { "A": 18.077889022432696, "B": 18.627129011951496, "C": 18.077889022432696 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.37523653320968153, "p50_s_raw": 0.11261826660484076, "p95_s_adjusted": 0.4140607553534209, "p95_s_raw": 0.13203037767671047 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 85.83309061783127, 90.19679304741126 ], "I": [ 59.88680163370944, 68.49583860207214 ], "capability": [ 73.72374153392347, 78.55486844196058 ], "composite": { "A": [ 17.720899810915583, 18.261473654252256 ], "B": [ 18.087561450267057, 18.993796215955186 ], "C": [ 17.656534367876468, 18.261473654252256 ] }, "seed": 36 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.97714437568723, "capability": 77.18992599407068, "intelligence": 64.40270761245412 }, "per_type": { "choice": { "accuracy": 0.836, "calibration": 92.09845493204818, "cc": 79.02674232448868, "cc_by_type": { "choice": 79.02674232448868 }, "cc_tiered": 79.48971421441038, "cc_tiered_open": 80.41823040534584, "cc_tiered_sealed": 78.56119802347493, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 79.02674232448868 }, "noul": { "accuracy": 0.784, "calibration": 87.6362174804601, "cc": 56.800000000000004, "cc_by_type": { "noul": 56.800000000000004 }, "cc_tiered": 55.80062290707754, "cc_tiered_open": 58.24775707384403, "cc_tiered_sealed": 53.35348874031106, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 56.800000000000004 }, "score": { "accuracy": 0.6853333333333333, "calibration": 90.19676071455342, "cc": 57.48579408299472, "cc_by_type": { "score": 57.48579408299472 }, "cc_tiered": 57.91778571587443, "cc_tiered_open": 58.2234599847005, "cc_tiered_sealed": 57.61211144704836, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 57.48579408299472 } }, "raw_on_A": { "calibration": 86.77546118929445, "capability": 77.43357682556581, "intelligence": 68.09169246183717 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 15.999272744012083, "Emphasis on Accuracy 60:20:20": 17.74116866920125, "Emphasis on Cost 20:20:60": 12.523731672975002, "Emphasis on Speed 20:60:20": 19.495440346386523, "Intelligence only": 21.20399692738122 }, "class": "unclassified", "open": "unknown", "licence": "not recorded", "repo": "https://huggingface.co/caiovicentino1/eikos-27b", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": null, "adapter_id": "images/eikos-27b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.8086253369272237, "acc_among_decisive": 0.98, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Igor L. / NInfer contributors", "axes": { "calibration": 84.18557302696732, "cost": 29.118887844204664, "intelligence": 51.800126951055375, "speed": 90.47734239273063 }, "calibration": { "parts": { "choice": { "ece_hard": 0.09587088648606537, "ece_score": 80.82582270278694, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 80.82582270278694 }, "noul": { "abstention_rate": 0.18328840970350402, "brier": 0.12070867053402283, "ece": 0.0947034093144267, "n": 371, "score": 81.05931813711467 }, "score": { "interval90_coverage": 0.962059620596206, "interval90_width": 0.4135049683830172, "n": 369, "nrps": 0.07983756609019925, "score": 90.67157824100035, "top_ece": 0.05336543454489691 } }, "score": 84.18557302696732 }, "capability": 67.99284998901135, "composite_ci95": { "A": [ 14.657219833459003, 18.09581018423065 ], "B": [ 14.41402694236696, 18.285046805435694 ], "C": [ 10.178624884346531, 15.973963875467245 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.2305174137931034 }, "display": "NInfer Qwen3.8-27B NVFP4", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 51.8273559598382, "I_sealed": 51.77289794227256, "base": 51.800126951055375, "excess": -2.5189296248726003, "gap": 0.0544580175656435, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 68.57305975575287, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 84.43243243243244, "hard": 56.59012268008808, "judge": 68.35860250494396, "standard": 84.93093344495604 } }, "open|noul": { "cc": 37.668046928916496, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 39.13043478260869, "judge": 30.000000000000004, "standard": 22.222222222222232 } }, "open|score": { "cc": 49.24096119484526, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 61.871723226731845, "hard": 32.99494797533653, "judge": 63.458675062497626, "standard": 54.0910358164409 } }, "sealed|choice": { "cc": 63.99439201930662, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 70.73387037173899, "hard": 56.730769230769226, "judge": 63.68715083798884, "standard": 75.61276019214189 } }, "sealed|noul": { "cc": 42.068015862163264, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 68.0, "hard": 20.388349514563096, "judge": 50.0, "standard": 60.56338028169015 } }, "sealed|score": { "cc": 49.25628594534781, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 68.47448220301372, "hard": 39.79827642333791, "judge": 53.88993870966854, "standard": 51.612727714053584 } } } }, "jevbench_score": 17.71600496541158, "key": "ninfer-qwen3.8-27b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 39, "ranked": true, "ranks": { "A": 39, "B": 38, "C": 40, "capability": 17 }, "scores": { "A": 17.71600496541158, "B": 17.686352887683018, "C": 13.20459060190698 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.26002668901346626, "p50_s_raw": 0.05501334450673312, "p95_s_adjusted": 0.3445464937365614, "p95_s_raw": 0.09727324686828069 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.72416061844064, 85.97779338206968 ], "I": [ 46.35923001218984, 56.308300363548845 ], "capability": [ 63.42455870096435, 70.41651644718259 ], "composite": { "A": [ 14.657219833459003, 18.09581018423065 ], "B": [ 14.41402694236696, 18.285046805435694 ], "C": [ 10.178624884346531, 15.973963875467245 ] }, "seed": 72 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 84.18557302696732, "capability": 67.99284998901135, "intelligence": 51.800126951055375 }, "per_type": { "choice": { "accuracy": 0.7266666666666667, "calibration": 80.82582270278694, "cc": 65.04457054081448, "cc_by_type": { "choice": 65.04457054081448 }, "cc_tiered": 66.28372588752974, "cc_tiered_open": 68.57305975575287, "cc_tiered_sealed": 63.99439201930662, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 65.04457054081448 }, "noul": { "accuracy": 0.7226666666666667, "calibration": 81.05931813711467, "cc": 44.53333333333334, "cc_by_type": { "noul": 44.53333333333334 }, "cc_tiered": 39.86803139553988, "cc_tiered_open": 37.668046928916496, "cc_tiered_sealed": 42.068015862163264, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 44.53333333333334 }, "score": { "accuracy": 0.6053333333333333, "calibration": 90.67157824100035, "cc": 49.00847977112264, "cc_by_type": { "score": 49.00847977112264 }, "cc_tiered": 49.24862357009654, "cc_tiered_open": 49.24096119484526, "cc_tiered_sealed": 49.25628594534781, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 49.00847977112264 } }, "raw_on_A": { "calibration": 79.59030598423057, "capability": 67.58012336905948, "intelligence": 55.569940753888396 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 15.726436779262448, "Emphasis on Accuracy 60:20:20": 16.41495956225673, "Emphasis on Cost 20:20:60": 12.713872701078419, "Emphasis on Speed 20:60:20": 19.536094431908698, "Intelligence only": 17.568730575960352 }, "class": "native-logit", "open": "yes", "licence": "Apache-2.0 (engine and submitted 27B artifact/base)", "repo": "https://github.com/igorls/ninfer", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": "Qwen/Qwen3.8-27B, NInfer NVFP4/FP8 artifact", "adapter_id": "images/ninfer-qwen3.8-27b/spec.json:ninfer_native.NInferNativeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.816711590296496, "acc_among_decisive": 0.8943894389438944, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "razorback16", "axes": { "calibration": 81.41345474033434, "cost": 28.51480924396573, "intelligence": 70.34621384425424, "speed": 72.55270827201397 }, "calibration": { "parts": { "choice": { "ece_hard": 0.1238179439505489, "ece_score": 75.23641120989022, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 75.23641120989022 }, "noul": { "abstention_rate": 0.04312668463611857, "brier": 0.06619006264403647, "ece": 0.07056572946447708, "n": 371, "score": 85.88685410710458 }, "score": { "interval90_coverage": 0.8455284552845529, "interval90_width": 0.06526648599819332, "n": 369, "nrps": 0.05970047185852581, "score": 83.11709890400827, "top_ece": 0.13897877503065434 } }, "score": 81.41345474033434 }, "capability": 75.87983429229429, "composite_ci95": { "A": [ 16.909474945094924, 17.502684272940023 ], "B": [ 17.624044058024868, 18.54119755452704 ], "C": [ 16.909474945094924, 17.502684272940023 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.24145693965517237 }, "display": "OpenJev (thinking, BF16)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 68.15800897274833, "I_sealed": 72.53441871576014, "base": 70.34621384425424, "excess": -6.949797385450049, "gap": -4.376409743011806, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 71.32537528109535, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 100.00000000000001, "hard": 67.9144385026738, "judge": 72.31377719182596, "standard": 62.32733361239012 } }, "open|noul": { "cc": 68.51345755693583, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 56.52173913043479, "judge": 80.0, "standard": 66.66666666666667 } }, "open|score": { "cc": 64.63519408021384, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 75.65890690670581, "hard": 83.20743211500854, "judge": 51.76582051786083, "standard": 41.282921940907926 } }, "sealed|choice": { "cc": 72.81145995047186, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 86.6972138053359, "hard": 66.4027149321267, "judge": 74.86033519553072, "standard": 75.61276019214189 } }, "sealed|noul": { "cc": 76.43903646714215, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 84.00000000000001, "hard": 61.165048543689316, "judge": 84.21052631578947, "standard": 91.54929577464787 } }, "sealed|score": { "cc": 68.35275972966642, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 85.29967322252621, "hard": 58.44270568724672, "judge": 72.99731091086852, "standard": 72.73258429627272 } } } }, "jevbench_score": 17.265218966202397, "key": "openjev-thinking", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 40, "ranked": true, "ranks": { "A": 40, "B": 36, "C": 30, "capability": 7 }, "scores": { "A": 17.265218966202397, "B": 18.156241294771178, "C": 17.265218966202397 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 1.8639664588030427, "p50_s_raw": 0.8569832294015214, "p95_s_adjusted": 2.980513406405225, "p95_s_raw": 1.4152567032026127 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 77.19017694535097, 83.84266027008114 ], "I": [ 65.12984274564428, 74.6441850225165 ], "capability": [ 71.81263659682128, 78.75991583946256 ], "composite": { "A": [ 16.909474945094924, 17.502684272940023 ], "B": [ 17.624044058024868, 18.54119755452704 ], "C": [ 16.909474945094924, 17.502684272940023 ] }, "seed": 81 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 81.41345474033434, "capability": 75.87983429229429, "intelligence": 70.34621384425424 }, "per_type": { "choice": { "accuracy": 0.7946666666666666, "calibration": 75.23641120989022, "cc": 73.74079933309964, "cc_by_type": { "choice": 73.74079933309964 }, "cc_tiered": 72.0684176157836, "cc_tiered_open": 71.32537528109535, "cc_tiered_sealed": 72.81145995047186, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 73.74079933309964 }, "noul": { "accuracy": 0.8826666666666667, "calibration": 85.88685410710458, "cc": 76.53333333333335, "cc_by_type": { "noul": 76.53333333333335 }, "cc_tiered": 72.47624701203898, "cc_tiered_open": 68.51345755693583, "cc_tiered_sealed": 76.43903646714215, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 76.53333333333335 }, "score": { "accuracy": 0.7786666666666666, "calibration": 83.11709890400827, "cc": 68.45852348469666, "cc_by_type": { "score": 68.45852348469666 }, "cc_tiered": 66.49397690494013, "cc_tiered_open": 64.63519408021384, "cc_tiered_sealed": 68.35275972966642, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 68.45852348469666 } }, "raw_on_A": { "calibration": 79.80009682824573, "capability": 76.44304334504619, "intelligence": 73.08598986184666 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 15.470816250462342, "Emphasis on Accuracy 60:20:20": 17.77278752746646, "Emphasis on Cost 20:20:60": 12.207984243040777, "Emphasis on Speed 20:60:20": 17.94233922790431, "Intelligence only": 22.879243501891544 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0", "repo": "https://github.com/razorback16/openjev", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "google/diffusiongemma-26b-a4b-it, BF16; OpenJev think=512", "adapter_id": "images/openjev-thinking/spec.json:openjev_thinking.ThinkingOpenJev", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.9568733153638814, "acc_among_decisive": 0.9323943661971831, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Zefan Cai (@Zefan_Cai)", "axes": { "calibration": 79.80507069734225, "cost": 33.05490478272432, "intelligence": 43.88591848753404, "speed": 74.39486349899005 }, "calibration": { "parts": { "choice": { "ece_hard": 0.0965793794449889, "ece_score": 80.68412411100222, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 80.68412411100222 }, "noul": { "abstention_rate": 0.18867924528301883, "brier": 0.1465943912444352, "ece": 0.11119868678055905, "n": 371, "score": 77.76026264388818 }, "score": { "interval90_coverage": 0.926829268292683, "interval90_width": 0.46047877145438126, "n": 369, "nrps": 0.10981645578089093, "score": 80.97082533713633, "top_ece": 0.13538351873819116 } }, "score": 79.80507069734225 }, "capability": 61.84549459243814, "composite_ci95": { "A": [ 12.01117001715265, 21.4767289933671 ], "B": [ 11.375313778606227, 21.19925799700656 ], "C": [ 8.341090289689344, 14.91439513428271 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.17041317733990147 }, "display": "Open-Jev 9B (Zefan Cai)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 45.51928930503784, "I_sealed": 42.25254767003025, "base": 43.88591848753404, "excess": 0.6933539925693402, "gap": 3.266741635007584, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 61.95574336697866, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 68.86486486486487, "hard": 64.1396665618119, "judge": 56.49307844429796, "standard": 62.32733361239012 } }, "open|noul": { "cc": 35.9399585921325, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 47.82608695652173, "judge": 10.000000000000009, "standard": 33.33333333333333 } }, "open|score": { "cc": 38.66216595600236, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 45.958227241534885, "hard": 49.02462444279136, "judge": 29.24271270673138, "standard": 28.41839821356458 } }, "sealed|choice": { "cc": 56.97677837170926, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 64.08247727440693, "hard": 48.076923076923066, "judge": 59.96275605214154, "standard": 66.7446729892844 } }, "sealed|noul": { "cc": 36.79892764867179, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 48.0, "hard": 20.388349514563096, "judge": 44.736842105263165, "standard": 52.112676056338024 } }, "sealed|score": { "cc": 32.98193698970971, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 51.23313360297058, "hard": 27.657644820832182, "judge": 32.23017056028181, "standard": 35.63257266497618 } } } }, "jevbench_score": 17.045611746249723, "key": "open-jev-zefan-9b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 41, "ranked": true, "ranks": { "A": 41, "B": 41, "C": 42, "capability": 31 }, "scores": { "A": 17.045611746249723, "B": 16.53766290698564, "C": 11.837230379340085 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 1.3850112770218401, "p50_s_raw": 0.6175056385109201, "p95_s_adjusted": 2.624583175848238, "p95_s_raw": 1.237291587924119 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 73.96736762912185, 81.45578416794778 ], "I": [ 37.82741036517579, 48.601933428170696 ], "capability": [ 56.87009089863343, 64.27945845500723 ], "composite": { "A": [ 12.01117001715265, 21.4767289933671 ], "B": [ 11.375313778606227, 21.19925799700656 ], "C": [ 8.341090289689344, 14.91439513428271 ] }, "seed": 79 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 79.80507069734225, "capability": 61.84549459243814, "intelligence": 43.88591848753404 }, "per_type": { "choice": { "accuracy": 0.6733333333333333, "calibration": 80.68412411100222, "cc": 58.22399893902218, "cc_by_type": { "choice": 58.22399893902218 }, "cc_tiered": 59.466260869343955, "cc_tiered_open": 61.95574336697866, "cc_tiered_sealed": 56.97677837170926, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 58.22399893902218 }, "noul": { "accuracy": 0.6933333333333334, "calibration": 77.76026264388818, "cc": 38.66666666666667, "cc_by_type": { "noul": 38.66666666666667 }, "cc_tiered": 36.369443120402146, "cc_tiered_open": 35.9399585921325, "cc_tiered_sealed": 36.79892764867179, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 38.66666666666667 }, "score": { "accuracy": 0.536, "calibration": 80.97082533713633, "cc": 35.27556488047482, "cc_by_type": { "score": 35.27556488047482 }, "cc_tiered": 35.82205147285603, "cc_tiered_open": 38.66216595600236, "cc_tiered_sealed": 32.98193698970971, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 35.27556488047482 } }, "raw_on_A": { "calibration": 76.09059647397694, "capability": 61.58634472209667, "intelligence": 47.0820929702164 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 15.193819551111247, "Emphasis on Accuracy 60:20:20": 15.024035626101695, "Emphasis on Cost 20:20:60": 13.257314615761418, "Emphasis on Speed 20:60:20": 18.03145694648765, "Intelligence only": 14.776357163022537 }, "class": "jev-rebuild", "open": "yes", "licence": "MIT (loader); Apache-2.0 (adapter and pinned Qwen base); CC0-1.0 public training projection", "repo": "https://github.com/Zefan-Cai/Open-Jev", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "Qwen3.5-9B plus rank-8 LoRA and trained scalar decision head", "adapter_id": "images/open-jev-zefan-9b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.8113207547169812, "acc_among_decisive": 0.8637873754152824, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Standard Thinking (myeongho12)", "axes": { "calibration": 84.01508092906352, "cost": 43.283250331904235, "intelligence": 32.80426604507879, "speed": 92.76442849252426 }, "calibration": { "parts": { "choice": { "ece_hard": 0.1098197498466637, "ece_score": 78.03605003066725, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 78.03605003066725 }, "noul": { "abstention_rate": 0.32345013477088946, "brier": 0.1399053250455738, "ece": 0.07067323668547182, "n": 371, "score": 85.86535266290564 }, "score": { "interval90_coverage": 0.907859078590786, "interval90_width": 0.4114724480578141, "n": 369, "nrps": 0.10288890604189731, "score": 88.14384009361767, "top_ece": 0.06711714604287465 } }, "score": 84.01508092906352 }, "capability": 58.40967348707115, "composite_ci95": { "A": [ 11.018270368438198, 24.24546849303509 ], "B": [ 9.568356102409696, 22.27960999090091 ], "C": [ 7.651576644748748, 16.837130897941034 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.07772456896551724 }, "display": "Standard One 8B (Standard Thinking)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 30.33450693574831, "I_sealed": 35.27402515440927, "base": 32.80426604507879, "excess": -7.512905861099208, "gap": -4.9395182186609645, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 49.295789751985026, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 53.2972972972973, "hard": 52.815350739226176, "judge": 44.62755438365195, "standard": 47.25826705734617 } }, "open|noul": { "cc": 8.598343685300202, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": 13.043478260869556, "judge": -30.000000000000004, "standard": 33.33333333333333 } }, "open|score": { "cc": 33.109387369959705, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 52.66883781130674, "hard": 20.17576470368565, "judge": 52.27846833393313, "standard": 20.443286035874163 } }, "sealed|choice": { "cc": 51.12536859237399, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 70.73387037173899, "hard": 39.42307692307691, "judge": 57.16945996275605, "standard": 55.65956398571254 } }, "sealed|noul": { "cc": 17.63077432402283, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 56.00000000000001, "hard": -12.621359223300976, "judge": 31.578947368421062, "standard": 38.0281690140845 } }, "sealed|score": { "cc": 37.06593254683102, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 56.312646632383355, "hard": 30.71497074829812, "judge": 39.8913069210783, "standard": 35.90643753974974 } } } }, "jevbench_score": 16.917108299009165, "key": "standardone-8b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 42, "ranked": true, "ranks": { "A": 42, "B": 43, "C": 43, "capability": 39 }, "scores": { "A": 16.917108299009165, "B": 15.107984144533011, "C": 11.747991874311923 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.21347399484366178, "p50_s_raw": 0.03173699742183089, "p95_s_adjusted": 0.24786326913163065, "p95_s_raw": 0.04893163456581533 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 77.9630533878547, 85.6304653046258 ], "I": [ 27.551137017862857, 38.18793043954382 ], "capability": [ 53.64856195871698, 60.83707776399651 ], "composite": { "A": [ 11.018270368438198, 24.24546849303509 ], "B": [ 9.568356102409696, 22.27960999090091 ], "C": [ 7.651576644748748, 16.837130897941034 ] }, "seed": 103 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 84.01508092906352, "capability": 58.40967348707115, "intelligence": 32.80426604507879 }, "per_type": { "choice": { "accuracy": 0.62, "calibration": 78.03605003066725, "cc": 51.40342733722988, "cc_by_type": { "choice": 51.40342733722988 }, "cc_tiered": 50.21057917217951, "cc_tiered_open": 49.295789751985026, "cc_tiered_sealed": 51.12536859237399, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 51.40342733722988 }, "noul": { "accuracy": 0.6026666666666667, "calibration": 85.86535266290564, "cc": 20.53333333333334, "cc_by_type": { "noul": 20.53333333333334 }, "cc_tiered": 13.114559004661515, "cc_tiered_open": 8.598343685300202, "cc_tiered_sealed": 17.63077432402283, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 20.53333333333334 }, "score": { "accuracy": 0.552, "calibration": 88.14384009361767, "cc": 36.63215814216161, "cc_by_type": { "score": 36.63215814216161 }, "cc_tiered": 35.08765995839536, "cc_tiered_open": 33.109387369959705, "cc_tiered_sealed": 37.06593254683102, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 36.63215814216161 } }, "raw_on_A": { "calibration": 79.00702916038831, "capability": 58.75187777403198, "intelligence": 38.49672638767566 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 15.03402443438624, "Emphasis on Accuracy 60:20:20": 12.868191666930013, "Emphasis on Cost 20:20:60": 14.585944618765962, "Emphasis on Speed 20:60:20": 18.769748652230398, "Intelligence only": 10.581584191343778 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 (jev-adapter server, LoRA and merged weights; base Ministral 3 8B Apache-2.0)", "repo": "https://huggingface.co/StandardThinking/StandardOne-8B", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": null, "adapter_id": "images/standardone-8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6765498652291105, "acc_among_decisive": 0.900398406374502, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Cloudflare", "axes": { "calibration": 91.57703482429629, "cost": 28.102157220114762, "intelligence": 59.08165501447566, "speed": 83.3998209667487 }, "calibration": { "parts": { "choice": { "ece_hard": 0.04191843750000011, "ece_score": 91.61631249999998, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 91.61631249999998 }, "noul": { "abstention_rate": 0.18400000000000005, "brier": 0.08179747616000003, "ece": 0.035773866666666626, "n": 375, "score": 92.84522666666668 }, "score": { "interval90_coverage": 0.92, "interval90_width": 0.31244444444444447, "n": 375, "nrps": 0.07581989387555554, "score": 90.26956530622223, "top_ece": 0.05939439999999997 } }, "score": 91.57703482429629 }, "capability": 75.32934491938597, "composite_ci95": { "A": [ 16.278742772255967, 17.01395343115317 ], "B": [ 16.406079295395777, 17.554401382209043 ], "C": [ 13.014738345348354, 17.01395343115317 ] }, "cost": { "basis": "latest published v1.5 row (v1.5.8 combined file) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied; self-hosted estimate, not measured hosted billing", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.24922681034482758 }, "display": "Clef (Cloudflare, Qwen3.8-27B post-train with a joint schema head, multimodal, measured on text)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 57.04595985248707, "I_sealed": 61.117350176464235, "base": 59.08165501447566, "excess": -6.644777966415406, "gap": -4.0713903239771625, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 76.21786468827244, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 77.35136835482857, "judge": 72.31377719182596, "standard": 69.8618668899121 } }, "open|noul": { "cc": 43.36853002070393, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 47.82608695652173, "judge": 30.000000000000004, "standard": 33.33333333333333 } }, "open|score": { "cc": 51.55148484848486, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 61.974848484848486, "hard": 50.55657407407409, "judge": 67.49000000000001, "standard": 24.421851851851862 } }, "sealed|choice": { "cc": 74.36979440941545, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 81.37609932747026, "hard": 69.9660633484163, "judge": 74.86033519553072, "standard": 78.93829289321346 } }, "sealed|noul": { "cc": 52.76571930304361, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 72.0, "hard": 32.038834951456316, "judge": 63.1578947368421, "standard": 69.01408450704226 } }, "sealed|score": { "cc": 56.216536816933655, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 56.72936170212767, "hard": 49.606985294117656, "judge": 66.35520958083832, "standard": 53.971218274111656 } } } }, "jevbench_score": 16.753692861922005, "key": "clef", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 43, "ranked": true, "ranks": { "A": 43, "B": 39, "C": 34, "capability": 8 }, "scores": { "A": 16.753692861922005, "B": 17.103724421252515, "C": 16.244762021167798 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.5763746980577708, "p50_s_raw": 0.21318734902888536, "p95_s_adjusted": 0.7930726912803948, "p95_s_raw": 0.3215363456401974 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 86.28138326498332, 91.94843594390738 ], "I": [ 53.626530822718266, 63.68048125704343 ], "capability": [ 70.47069839174009, 77.24671214753253 ], "composite": { "A": [ 16.278742772255967, 17.01395343115317 ], "B": [ 16.406079295395777, 17.554401382209043 ], "C": [ 13.014738345348354, 17.01395343115317 ] }, "seed": 20 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 91.57703482429629, "capability": 75.32934491938597, "intelligence": 59.08165501447566 }, "per_type": { "choice": { "accuracy": 0.8093333333333333, "calibration": 91.61631249999998, "cc": 75.61645652359255, "cc_by_type": { "choice": 75.61645652359255 }, "cc_tiered": 75.29382954884395, "cc_tiered_open": 76.21786468827244, "cc_tiered_sealed": 74.36979440941545, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 75.61645652359255 }, "noul": { "accuracy": 0.768, "calibration": 92.84522666666668, "cc": 53.6, "cc_by_type": { "noul": 53.6 }, "cc_tiered": 48.06712466187377, "cc_tiered_open": 43.36853002070393, "cc_tiered_sealed": 52.76571930304361, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 53.6 }, "score": { "accuracy": 0.6613333333333333, "calibration": 90.26956530622223, "cc": 54.4725614035088, "cc_by_type": { "score": 54.4725614035088 }, "cc_tiered": 53.88401083270926, "cc_tiered_open": 51.55148484848486, "cc_tiered_sealed": 56.216536816933655, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 54.4725614035088 } }, "raw_on_A": { "calibration": 88.38059707442306, "capability": 75.82665436300499, "intelligence": 63.27271165158693 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 14.692533500484354, "Emphasis on Accuracy 60:20:20": 16.05926983891725, "Emphasis on Cost 20:20:60": 11.641983750272722, "Emphasis on Speed 20:60:20": 17.850768289881103, "Intelligence only": 18.663451480355018 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/Cloudflare/clef", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100 80GB), offline (HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE), credential-free, server bound to 127.0.0.1, harness on the same pod over loopback HTTP; Lium pod 08a1ff4a-248a-4ff2-948e-ea558f99e808 (huid cosmic-hawk-4f, node calm-fox-65), 1x NVIDIA H100 80GB HBM3, United States, USD 1.30/h", "underlying": "Qwen/Qwen3.8-27B", "adapter_id": "/home/flori/jobs/jevbench-add-requests-20260919/r39/outputs/meta/specs/clef.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "alt": { "label": "Workers AI price scenario (latency unmeasured)", "note": "Cloudflare standard launch input price; same self-hosted accuracy and adjusted H100 latency held fixed for this price-only scenario. No Workers AI latency was measured; this is not a hosted API result and does not set official Capability eligibility. Official self-hosted row uses the base-model reference price.", "usd_per_1000": 0.1424153201970443, "axes": { "intelligence": 67.9181689327805, "calibration": 86.72295444340945, "speed": 79.86099006822079, "cost": 35.393298680703595 }, "pricing_only": true }, "model_pin": "Cloudflare/clef@2f3de3dd85f379784083b0814d997ab627200f0c", "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.816, "acc_among_decisive": 0.9411764705882353, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "WilfLin", "axes": { "calibration": 85.03288206087805, "cost": 47.592235403520306, "intelligence": 29.251131763478327, "speed": 93.62779491136789 }, "calibration": { "parts": { "choice": { "ece_hard": 0.05599448584772476, "ece_score": 88.80110283045505, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 88.80110283045505 }, "noul": { "abstention_rate": 0.47439353099730464, "brier": 0.15464738864390087, "ece": 0.10676843098382341, "n": 371, "score": 78.64631380323533 }, "score": { "interval90_coverage": 0.975609756097561, "interval90_width": 0.6402439024390243, "n": 369, "nrps": 0.10255360461757664, "score": 87.65122954894377, "top_ece": 0.07221090220177398 } }, "score": 85.03288206087805 }, "capability": 57.14200691217819, "composite_ci95": { "A": [ 10.554100896173676, 22.576213316295057 ], "B": [ 8.924509602976766, 20.049092209565615 ], "C": [ 7.329236733453941, 15.67792591409379 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (proxy tokens, I-2): exact base-model market reference", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.05583743842364531 }, "display": "JEV Qwen3.5-9B Base NVFP4", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 30.95041263176777, "I_sealed": 27.551850895188885, "base": 29.251131763478327, "excess": 0.8251740941406402, "gap": 3.398561736578884, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 55.43165802161466, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 53.2972972972973, "hard": 47.153192827933324, "judge": 52.537903757415954, "standard": 77.39640016743407 } }, "open|noul": { "cc": 3.6694271911663243, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": -21.739130434782606, "judge": 10.000000000000009, "standard": 11.111111111111116 } }, "open|score": { "cc": 33.750152682522334, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 58.19655618503351, "hard": 33.2215341463409, "judge": 26.48323152009081, "standard": 33.48456974727688 } }, "sealed|choice": { "cc": 47.891397457772825, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 66.74303451333975, "hard": 41.96832579185519, "judge": 45.065176908752335, "standard": 54.55105308535534 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 24.0, "hard": -26.21359223300971, "judge": -18.42105263157895, "standard": -9.859154929577462 } }, "sealed|score": { "cc": 34.764155227793836, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 59.111326959327606, "hard": 22.60553224460797, "judge": 38.31832027118539, "standard": 41.57656776331136 } } } }, "jevbench_score": 15.97555620862577, "key": "jev-qwen3.5-9b-base-nvfp4", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 44, "ranked": true, "ranks": { "A": 44, "B": 44, "C": 44, "capability": 43 }, "scores": { "A": 15.97555620862577, "B": 13.864517363543307, "C": 11.094136255990119 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.19534559999592602, "p50_s_raw": 0.02267279999796301, "p95_s_adjusted": 0.22203266686410641, "p95_s_raw": 0.03601633343205321 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.95850301005234, 86.17596383374455 ], "I": [ 24.784047457667274, 33.682401804173935 ], "capability": [ 52.832944781851424, 58.83201835103813 ], "composite": { "A": [ 10.554100896173676, 22.576213316295057 ], "B": [ 8.924509602976766, 20.049092209565615 ], "C": [ 7.329236733453941, 15.67792591409379 ] }, "seed": 47 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 85.03288206087805, "capability": 57.14200691217819, "intelligence": 29.251131763478327 }, "per_type": { "choice": { "accuracy": 0.608, "calibration": 88.80110283045505, "cc": 49.86879872682661, "cc_by_type": { "choice": 49.86879872682661 }, "cc_tiered": 51.66152773969374, "cc_tiered_open": 55.43165802161466, "cc_tiered_sealed": 47.891397457772825, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 49.86879872682661 }, "noul": { "accuracy": 0.464, "calibration": 78.64631380323533, "cc": -7.199999999999996, "cc_by_type": { "noul": -7.199999999999996 }, "cc_tiered": 1.8347135955831622, "cc_tiered_open": 3.6694271911663243, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.5413333333333333, "calibration": 87.65122954894377, "cc": 35.19517866359504, "cc_by_type": { "score": 35.19517866359504 }, "cc_tiered": 34.25715395515809, "cc_tiered_open": 33.750152682522334, "cc_tiered_sealed": 34.764155227793836, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 35.19517866359504 } }, "raw_on_A": { "calibration": 80.94123658739136, "capability": 56.72695595498426, "intelligence": 32.51267532257717 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 14.120527794466737, "Emphasis on Accuracy 60:20:20": 11.548484256330752, "Emphasis on Cost 20:20:60": 14.368613309854894, "Emphasis on Speed 20:60:20": 17.771756183840008, "Intelligence only": 9.070269959573535 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 entrant and upstream checkpoint", "repo": "https://huggingface.co/WIlfLin/JEV-Qwen3.5-9B-Base-NVFP4", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": "Qwen3.5-9B Base NVFP4 with compact BF16 decision head", "adapter_id": "images/jev-qwen3.5-9b-base-nvfp4/spec.json:jev_choice_http.Adapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5256064690026954, "acc_among_decisive": 0.8923076923076924, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Theodore Lee (TheoLeeCJ)", "axes": { "calibration": 83.59960184071504, "cost": 63.07359395281681, "intelligence": 27.10113669038628, "speed": 90.76560119421038 }, "calibration": { "parts": { "choice": { "ece_hard": 0.10233013188094879, "ece_score": 79.53397362381024, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 79.53397362381024 }, "noul": { "abstention_rate": 0.38274932614555257, "brier": 0.1432523071034433, "ece": 0.08378278501118164, "n": 371, "score": 83.24344299776368 }, "score": { "interval90_coverage": 0.9701897018970189, "interval90_width": 0.6594399277326105, "n": 369, "nrps": 0.1212670700676031, "score": 88.0213889005712, "top_ece": 0.059152575960486446 } }, "score": 83.59960184071504 }, "capability": 55.35036926555066, "composite_ci95": { "A": [ 9.244572028428712, 23.18413753262927 ], "B": [ 7.522852111652043, 19.995799931382237 ], "C": [ 6.419841686408828, 16.100095508770327 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01701689039408867 }, "display": "SemIf, formerly OpenJev (Qwen3.5-4B, TheoLeeCJ)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 30.391025070826444, "I_sealed": 23.811248309946116, "base": 27.10113669038628, "excess": 4.006389118442085, "gap": 6.579776760880328, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 49.58254501782712, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864865, "hard": 35.8288770053476, "judge": 60.44825313117996, "standard": 47.25826705734617 } }, "open|noul": { "cc": 14.439613526570046, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 13.043478260869556, "judge": -9.999999999999998, "standard": 11.111111111111116 } }, "open|score": { "cc": 27.150916668082164, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 62.363221487255814, "hard": 12.79441887527647, "judge": 38.365166132705816, "standard": 21.436385647171228 } }, "sealed|choice": { "cc": 45.71562508055857, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 61.4219200354741, "hard": 39.93212669683257, "judge": 45.99627560521416, "standard": 49.008498583569406 } }, "sealed|noul": { "cc": 2.17292780700555, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 43.99999999999999, "hard": -22.330097087378643, "judge": 2.6315789473684292, "standard": 29.5774647887324 } }, "sealed|score": { "cc": 23.545192042274238, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 63.77035344302047, "hard": 13.542770599427534, "judge": 22.665102695300575, "standard": 24.757588248055008 } } } }, "jevbench_score": 15.517179498801362, "key": "semif-qwen3.5-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 45, "ranked": true, "ranks": { "A": 45, "B": 45, "C": 45, "capability": 46 }, "scores": { "A": 15.517179498801362, "B": 13.04205072010579, "C": 10.775819096389835 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.25408456099685284, "p50_s_raw": 0.05204228049842641, "p95_s_adjusted": 0.32996023471932856, "p95_s_raw": 0.08998011735966428 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.25331982816681, 85.80759674871754 ], "I": [ 22.076409045997522, 31.830294818443246 ], "capability": [ 51.05521406162153, 58.22417544189523 ], "composite": { "A": [ 9.244572028428712, 23.18413753262927 ], "B": [ 7.522852111652043, 19.995799931382237 ], "C": [ 6.419841686408828, 16.100095508770327 ] }, "seed": 99 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 83.59960184071504, "capability": 55.35036926555066, "intelligence": 27.10113669038628 }, "per_type": { "choice": { "accuracy": 0.5893333333333334, "calibration": 79.53397362381024, "cc": 47.48159866619932, "cc_by_type": { "choice": 47.48159866619932 }, "cc_tiered": 47.64908504919285, "cc_tiered_open": 49.58254501782712, "cc_tiered_sealed": 45.71562508055857, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 47.48159866619932 }, "noul": { "accuracy": 0.552, "calibration": 83.24344299776368, "cc": 10.40000000000001, "cc_by_type": { "noul": 10.40000000000001 }, "cc_tiered": 8.306270666787798, "cc_tiered_open": 14.439613526570046, "cc_tiered_sealed": 2.17292780700555, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 10.40000000000001 }, "score": { "accuracy": 0.49333333333333335, "calibration": 88.0213889005712, "cc": 25.84341933063762, "cc_by_type": { "score": 25.84341933063762 }, "cc_tiered": 25.3480543551782, "cc_tiered_open": 27.150916668082164, "cc_tiered_sealed": 23.545192042274238, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 25.84341933063762 } }, "raw_on_A": { "calibration": 83.04331750428547, "capability": 58.219039699698534, "intelligence": 33.3947618951116 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 13.820858864848555, "Emphasis on Accuracy 60:20:20": 10.67791560389241, "Emphasis on Cost 20:20:60": 15.384875177789313, "Emphasis on Speed 20:60:20": 17.119455792629843, "Intelligence only": 7.962006198162448 }, "class": "jev-rebuild", "open": "yes", "licence": "MIT (code); Qwen3.5 weights Apache-2.0", "repo": "https://github.com/TheoLeeCJ/openjev", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": "Qwen/Qwen3.5-4B (frozen, BF16)", "adapter_id": "images/semif-qwen3.5-4b/spec.json:remote_inproc_v15.RemoteInprocV15Adapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6172506738544474, "acc_among_decisive": 0.9039301310043668, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Reza Sayar", "axes": { "calibration": 88.50232899798883, "cost": 28.231834531826323, "intelligence": 43.77513601921869, "speed": 72.98222461463935 }, "calibration": { "parts": { "choice": { "ece_hard": 0.08046726298491785, "ece_score": 83.90654740301643, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 83.90654740301643 }, "noul": { "abstention_rate": 0.3045822102425876, "brier": 0.11954030846459725, "ece": 0.04318003176619355, "n": 371, "score": 91.36399364676129 }, "score": { "interval90_coverage": 0.964769647696477, "interval90_width": 0.4652213188798555, "n": 369, "nrps": 0.08597620729262767, "score": 90.23644594418879, "top_ece": 0.0546474369117983 } }, "score": 88.50232899798883 }, "capability": 66.13873250860377, "composite_ci95": { "A": [ 8.55065455714213, 14.587291870035246 ], "B": [ 8.22423019498041, 14.527264406104972 ], "C": [ 5.9379545535709255, 10.130063798635586 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: documented hosted-model estimate. Frozen market reference qwen/qwen3.8-27b.", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.2467585344827586 }, "display": "Bev / Bonsai 27B", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 46.71700087936099, "I_sealed": 40.833271159076396, "base": 43.77513601921869, "excess": 3.310342077846348, "gap": 5.883729720284592, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 63.31036599788361, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675674, "hard": 47.153192827933324, "judge": 72.31377719182596, "standard": 77.39640016743407 } }, "open|noul": { "cc": 30.23947550034507, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 39.13043478260869, "judge": 10.000000000000009, "standard": 22.222222222222232 } }, "open|score": { "cc": 46.6011611398543, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 58.472258307241056, "hard": 30.60094415356245, "judge": 62.977497036063276, "standard": 48.101542684431145 } }, "sealed|choice": { "cc": 56.58157537232299, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 66.74303451333975, "hard": 49.09502262443438, "judge": 53.44506517690876, "standard": 71.17871659071314 } }, "sealed|noul": { "cc": 20.312021130359053, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 52.0, "hard": -0.9708737864077666, "judge": 26.315789473684205, "standard": 38.0281690140845 } }, "sealed|score": { "cc": 45.60621697454716, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 67.9108636782132, "hard": 33.06846017805201, "judge": 52.300237792599205, "standard": 49.48837598862635 } } } }, "jevbench_score": 11.7393704396132, "key": "bev-bonsai-27b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 46, "ranked": true, "ranks": { "A": 46, "B": 46, "C": 46, "capability": 21 }, "scores": { "A": 11.7393704396132, "B": 11.515061943796129, "C": 8.152340583064722 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 2.1222116896882652, "p50_s_raw": 0.9861058448441327, "p95_s_adjusted": 2.371312748454511, "p95_s_raw": 1.1106563742272555 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 81.75270117761126, 88.77503324811279 ], "I": [ 38.25187875678337, 48.17301489659999 ], "capability": [ 60.910547743797885, 67.5609085693478 ], "composite": { "A": [ 8.55065455714213, 14.587291870035246 ], "B": [ 8.22423019498041, 14.527264406104972 ], "C": [ 5.9379545535709255, 10.130063798635586 ] }, "seed": 17 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 88.50232899798883, "capability": 66.13873250860377, "intelligence": 43.77513601921869 }, "per_type": { "choice": { "accuracy": 0.672, "calibration": 83.90654740301643, "cc": 58.053484648977374, "cc_by_type": { "choice": 58.053484648977374 }, "cc_tiered": 59.9459706851033, "cc_tiered_open": 63.31036599788361, "cc_tiered_sealed": 56.58157537232299, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 58.053484648977374 }, "noul": { "accuracy": 0.6293333333333333, "calibration": 91.36399364676129, "cc": 25.86666666666666, "cc_by_type": { "noul": 25.86666666666666 }, "cc_tiered": 25.27574831535206, "cc_tiered_open": 30.23947550034507, "cc_tiered_sealed": 20.312021130359053, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 25.86666666666666 }, "score": { "accuracy": 0.6, "calibration": 90.23644594418879, "cc": 45.43825770414722, "cc_by_type": { "score": 45.43825770414722 }, "cc_tiered": 46.103689057200725, "cc_tiered_open": 46.6011611398543, "cc_tiered_sealed": 45.60621697454716, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 45.43825770414722 } }, "raw_on_A": { "calibration": 80.9895865212274, "capability": 63.89601132732785, "intelligence": 46.8024361334283 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 10.186877852314403, "Emphasis on Accuracy 60:20:20": 10.385148364433647, "Emphasis on Cost 20:20:60": 8.555937486887114, "Emphasis on Speed 20:60:20": 12.296000304326956, "Intelligence only": 10.697460589059927 }, "class": "system-one-open", "open": "yes", "licence": "Not independently recorded", "repo": "https://huggingface.co/Reza2kn/Bev", "gpu": "H100", "endpoint_condition": "evaluator-owned H100 GPU", "underlying": "Reza2kn/Bev @ f01edf952e3f2da8147aa08ec6d6229c863d5d19. Unchanged Prism ML Ternary-Bonsai-2-27B PQ2_0 quantization of Qwen3.8-27B, with selected-token readout; not a fine-tune. GGUF SHA-256 3907dc1658db1f78a9826bf8d5bcb8dc65db0d466388937af57f2294fae62ec1. H100; CUDA backend compiled from the author-pinned source 9a9394a895b96003ca842a6041cb28ac49a108f7 with the author patch, rather than their prebuilt binary. The wire shim drops noul criteria because the submitted schema rejects them; that loses option descriptions for noul.", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6954177897574124, "acc_among_decisive": 0.9147286821705426, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Zhengxu Yu", "axes": { "calibration": 89.9210925231847, "cost": 28.35809912639418, "intelligence": 43.28549422942267, "speed": 68.17671225113584 }, "calibration": { "parts": { "choice": { "ece_hard": 0.054566021834308726, "ece_score": 89.08679563313827, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 89.08679563313827 }, "noul": { "abstention_rate": 0.33692722371967654, "brier": 0.09143307684062531, "ece": 0.03620776305464309, "n": 371, "score": 92.75844738907139 }, "score": { "interval90_coverage": 0.981029810298103, "interval90_width": 0.534327009936766, "n": 369, "nrps": 0.08453630674044608, "score": 87.91803454734445, "top_ece": 0.07855150115633251 } }, "score": 89.9210925231847 }, "capability": 66.60329337630368, "composite_ci95": { "A": [ 8.746773766553591, 14.766757637183968 ], "B": [ 8.426311974704314, 14.74912210872967 ], "C": [ 6.074148448995549, 10.25469280359998 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.2443787068965517 }, "display": "LitJev (Qwen3.8-27B)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 41.98760951977819, "I_sealed": 44.583378939067146, "base": 43.28549422942267, "excess": -5.1691570617272, "gap": -2.595769419288956, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 68.57906399743361, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 84.43243243243244, "hard": 64.1396665618119, "judge": 68.35860250494396, "standard": 69.8618668899121 } }, "open|noul": { "cc": 12.532781228433404, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 4.347826086956519, "judge": 0.0, "standard": 11.111111111111116 } }, "open|score": { "cc": 44.85098333346756, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 58.30288571369996, "hard": 36.97174671674506, "judge": 54.42696015026953, "standard": 39.519540151593404 } }, "sealed|choice": { "cc": 68.65215618863985, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 81.37609932747026, "hard": 60.29411764705882, "judge": 66.4804469273743, "standard": 82.26382559428504 } }, "sealed|noul": { "cc": 24.95501594133015, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 68.0, "hard": -0.9708737864077666, "judge": 28.947368421052634, "standard": 49.29577464788733 } }, "sealed|score": { "cc": 40.142964687231455, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 64.90525292252141, "hard": 32.24057461852555, "judge": 45.24652806264088, "standard": 35.91125564388415 } } } }, "jevbench_score": 11.458886170296882, "key": "litjev", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 47, "ranked": true, "ranks": { "A": 47, "B": 47, "C": 47, "capability": 19 }, "scores": { "A": 11.458886170296882, "B": 11.238393678896292, "C": 7.957559840483947 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 3.0960148913785814, "p50_s_raw": 1.4730074456892908, "p95_s_adjusted": 4.9150250315666195, "p95_s_raw": 2.3825125157833096 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 84.4774023239196, 89.78804799226427 ], "I": [ 38.555280103835365, 48.498383980718195 ], "capability": [ 62.48585734643129, 68.14178334863401 ], "composite": { "A": [ 8.746773766553591, 14.766757637183968 ], "B": [ 8.426311974704314, 14.74912210872967 ], "C": [ 6.074148448995549, 10.25469280359998 ] }, "seed": 61 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.9210925231847, "capability": 66.60329337630368, "intelligence": 43.28549422942267 }, "per_type": { "choice": { "accuracy": 0.7613333333333333, "calibration": 89.08679563313827, "cc": 69.47794208197946, "cc_by_type": { "choice": 69.47794208197946 }, "cc_tiered": 68.61561009303674, "cc_tiered_open": 68.57906399743361, "cc_tiered_sealed": 68.65215618863985, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 69.47794208197946 }, "noul": { "accuracy": 0.64, "calibration": 92.75844738907139, "cc": 28.000000000000004, "cc_by_type": { "noul": 28.000000000000004 }, "cc_tiered": 18.743898584881777, "cc_tiered_open": 12.532781228433404, "cc_tiered_sealed": 24.95501594133015, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 28.000000000000004 }, "score": { "accuracy": 0.5626666666666666, "calibration": 87.91803454734445, "cc": 41.34354803654359, "cc_by_type": { "score": 41.34354803654359 }, "cc_tiered": 42.496974010349504, "cc_tiered_open": 44.85098333346756, "cc_tiered_sealed": 40.142964687231455, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 41.34354803654359 } }, "raw_on_A": { "calibration": 89.32478284904047, "capability": 69.19315337175932, "intelligence": 49.06152389447817 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 9.902804220753023, "Emphasis on Accuracy 60:20:20": 10.109112973826948, "Emphasis on Cost 20:20:60": 8.396442160471732, "Emphasis on Speed 20:60:20": 11.774987261719835, "Intelligence only": 10.435213933727047 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 (code); Apache-2.0 base weights", "repo": "https://github.com/zhengxuyu/litjev", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "Qwen/Qwen3.8-27B, frozen, read at the output head", "adapter_id": "images/litjev/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6630727762803235, "acc_among_decisive": 0.975609756097561, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Amith Chandrappa (amithgc)", "axes": { "calibration": 85.84286362718517, "cost": 59.269627658320445, "intelligence": 24.145363218466134, "speed": 86.10068849774444 }, "calibration": { "parts": { "choice": { "ece_hard": 0.039143749999999984, "ece_score": 92.17125, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 92.17125 }, "noul": { "abstention_rate": 0.648, "brier": 0.14289767218666674, "ece": 0.09039973333333337, "n": 375, "score": 81.92005333333331 }, "score": { "interval90_coverage": 0.96, "interval90_width": 0.6557777777777777, "n": 375, "nrps": 0.13420838236888896, "score": 83.4372875482222, "top_ece": 0.0985229333333334 } }, "score": 85.84286362718517 }, "capability": 54.99411342282565, "composite_ci95": { "A": [ 6.925385954427524, 15.24148889625826 ], "B": [ 5.568022650808403, 12.90281361284615 ], "C": [ 4.8092958016857805, 10.584367289068235 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.022786576354679802 }, "display": "local-jev Qwen3.5-4B", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 26.448995848149686, "I_sealed": 21.841730588782582, "base": 24.145363218466134, "excess": 2.03387761692886, "gap": 4.607265259367104, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 52.23659737153346, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 80.54054054054055, "hard": 41.49103491664046, "judge": 60.44825313117996, "standard": 47.25826705734617 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 28.57142857142858, "hard": -56.52173913043479, "judge": -30.000000000000004, "standard": -33.333333333333336 } }, "open|score": { "cc": 27.1103901729156, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 54.470606060606066, "hard": 15.197777777777777, "judge": 40.368135593220345, "standard": 17.36888888888889 } }, "sealed|choice": { "cc": 46.955426808141326, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 61.4219200354741, "hard": 42.477375565610856, "judge": 45.99627560521416, "standard": 50.1170094839266 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 12.00000000000001, "hard": -70.87378640776699, "judge": -18.42105263157895, "standard": -15.492957746478876 } }, "sealed|score": { "cc": 18.56976495820642, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 56.15659574468085, "hard": 12.02813725490197, "judge": 13.700838323353281, "standard": 20.162994923857813 } } } }, "jevbench_score": 11.438161297684031, "key": "localjev-qwen3.5-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 48, "ranked": true, "ranks": { "A": 48, "B": 49, "C": 48, "capability": 47 }, "scores": { "A": 11.438161297684031, "B": 9.482181005106208, "C": 7.943167567836132 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.39534114915877583, "p50_s_raw": 0.1226705745793879, "p95_s_adjusted": 0.6208106089383364, "p95_s_raw": 0.2354053044691682 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 79.50985112092962, 86.15050973358518 ], "I": [ 19.830618866047317, 27.13596492581447 ], "capability": [ 50.794515955411754, 55.552509505023664 ], "composite": { "A": [ 6.925385954427524, 15.24148889625826 ], "B": [ 5.568022650808403, 12.90281361284615 ], "C": [ 4.8092958016857805, 10.584367289068235 ] }, "seed": 62 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 85.84286362718517, "capability": 54.99411342282565, "intelligence": 24.145363218466134 }, "per_type": { "choice": { "accuracy": 0.6026666666666667, "calibration": 92.17125, "cc": 49.186741566647385, "cc_by_type": { "choice": 49.186741566647385 }, "cc_tiered": 49.596012089837394, "cc_tiered_open": 52.23659737153346, "cc_tiered_sealed": 46.955426808141326, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 49.186741566647385 }, "noul": { "accuracy": 0.34933333333333333, "calibration": 81.92005333333331, "cc": -30.133333333333333, "cc_by_type": { "noul": -30.133333333333333 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.45066666666666666, "calibration": 83.4372875482222, "cc": 22.259017543859695, "cc_by_type": { "score": 22.259017543859695 }, "cc_tiered": 22.84007756556101, "cc_tiered_open": 27.1103901729156, "cc_tiered_sealed": 18.56976495820642, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 22.259017543859695 } }, "raw_on_A": { "calibration": 81.40355743693019, "capability": 54.83642941083503, "intelligence": 28.26930138473988 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 10.008247462291514, "Emphasis on Accuracy 60:20:20": 7.6341780052667865, "Emphasis on Cost 20:20:60": 11.24977392613748, "Emphasis on Speed 20:60:20": 12.519995559254454, "Intelligence only": 5.630684842640914 }, "class": "jev-rebuild", "open": "yes", "licence": "MIT code; Apache-2.0 Qwen weights", "repo": "https://github.com/amithgc/local-jev", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen/Qwen3.5-4B text model, zero-shot next-token letter probabilities", "adapter_id": "images/localjev-qwen3.5-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.352, "acc_among_decisive": 0.9924242424242424, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Sean Goedecke", "axes": { "calibration": 30.16473321941294, "cost": 44.96855803039237, "intelligence": 29.271508779158296, "speed": 90.8867694864598 }, "calibration": { "parts": { "choice": { "ece_hard": 0.5207165079484644, "ece_score": 0.0, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 0.0 }, "noul": { "abstention_rate": 0.07547169811320753, "brier": 0.26938609640716954, "ece": 0.26647263490315864, "n": 371, "score": 46.70547301936827 }, "score": { "interval90_coverage": 0.5718157181571816, "interval90_width": 0.08897922312556458, "n": 369, "nrps": 0.18780642533502573, "score": 43.78872663887056, "top_ece": 0.46820952094378154 } }, "score": 30.16473321941294 }, "capability": 29.718120999285617, "composite_ci95": { "A": [ 5.879902468266102, 15.68532110290842 ], "B": [ 5.313278874085685, 14.848330476734906 ], "C": [ 4.08326560296257, 10.892584099241958 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.06829381896551724 }, "display": "system-one (Qwen3-8B, Sean Goedecke)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 32.67027386076885, "I_sealed": 25.87274369754774, "base": 29.271508779158296, "excess": 4.224142520782868, "gap": 6.797530163221111, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 45.21508118936821, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 61.08108108108109, "hard": 30.16671909405474, "judge": 48.58272907053395, "standard": 62.32733361239012 } }, "open|noul": { "cc": 28.967563837129056, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 30.434782608695656, "judge": 19.999999999999996, "standard": 11.111111111111116 } }, "open|score": { "cc": 23.8281765558093, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 65.82115115650328, "hard": 4.701913776879685, "judge": 35.713306286008326, "standard": 23.25652021802299 } }, "sealed|choice": { "cc": 35.384978952910906, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 54.77052693814206, "hard": 19.57013574660632, "judge": 35.7541899441341, "standard": 56.76807488606972 } }, "sealed|noul": { "cc": 28.751778735776956, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 72.0, "hard": -2.9126213592232997, "judge": 44.736842105263165, "standard": 46.47887323943662 } }, "sealed|score": { "cc": 13.48147340395537, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 53.14233360647993, "hard": 0.26200782926150135, "judge": 16.1186769737661, "standard": 16.134169097364737 } } } }, "jevbench_score": 11.027723483913586, "key": "system-one-sg", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 49, "ranked": true, "ranks": { "A": 49, "B": 48, "C": 49, "capability": 85 }, "scores": { "A": 11.027723483913586, "B": 10.28901604793032, "C": 7.658141308273324 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.24992560098180547, "p50_s_raw": 0.04996280049090274, "p95_s_adjusted": 0.3262212941306643, "p95_s_raw": 0.08811064706533216 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 26.715281574276638, 33.82082873066771 ], "I": [ 22.79600693038093, 33.62231882810145 ], "capability": [ 25.123779554066562, 33.19293706149128 ], "composite": { "A": [ 5.879902468266102, 15.68532110290842 ], "B": [ 5.313278874085685, 14.848330476734906 ], "C": [ 4.08326560296257, 10.892584099241958 ] }, "seed": 105 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 30.16473321941294, "capability": 29.718120999285617, "intelligence": 29.271508779158296 }, "per_type": { "choice": { "accuracy": 0.5133333333333333, "calibration": 0.0, "cc": 37.76228413364529, "cc_by_type": { "choice": 37.76228413364529 }, "cc_tiered": 40.30003007113956, "cc_tiered_open": 45.21508118936821, "cc_tiered_sealed": 35.384978952910906, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 37.76228413364529 }, "noul": { "accuracy": 0.6666666666666666, "calibration": 46.70547301936827, "cc": 33.33333333333333, "cc_by_type": { "noul": 33.33333333333333 }, "cc_tiered": 28.859671286453008, "cc_tiered_open": 28.967563837129056, "cc_tiered_sealed": 28.751778735776956, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 33.33333333333333 }, "score": { "accuracy": 0.45066666666666666, "calibration": 43.78872663887056, "cc": 16.56590736182537, "cc_by_type": { "score": 16.56590736182537 }, "cc_tiered": 18.654824979882335, "cc_tiered_open": 23.8281765558093, "cc_tiered_sealed": 13.48147340395537, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 16.56590736182537 } }, "raw_on_A": { "calibration": 29.76734864606182, "capability": 30.080860832336057, "intelligence": 30.394373018610295 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 12.338652673779826, "Emphasis on Accuracy 60:20:20": 10.21232996468921, "Emphasis on Cost 20:20:60": 12.38939107483353, "Emphasis on Speed 20:60:20": 15.503079386401165, "Intelligence only": 8.114714247763947 }, "class": "jev-rebuild", "open": "yes", "licence": "no licence file in the repository as of 19 Sep; Qwen3 weights Apache-2.0", "repo": "https://github.com/sgoedecke/system-one", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": "Qwen/Qwen3-8B (frozen, BF16), the model of the author's demos", "adapter_id": "images/system-one-sg/spec.json:remote_inproc_v15.RemoteInprocV15Adapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.9245283018867925, "acc_among_decisive": 0.7288629737609329, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Jared Palmer", "axes": { "calibration": 70.7561729087212, "cost": 40.42054386073457, "intelligence": 29.26217941280341, "speed": 88.97143473610706 }, "calibration": { "parts": { "choice": { "ece_hard": 0.14254951638954755, "ece_score": 71.49009672209048, "mean_tvd": null, "n": 736, "n_ece_hard": 306, "n_tvd": 0, "score": 71.49009672209048 }, "noul": { "abstention_rate": 0.23450134770889486, "brier": 0.18970835579514836, "ece": 0.16199460916442043, "n": 371, "score": 67.60107816711592 }, "score": { "interval90_coverage": 0.8233695652173914, "interval90_width": 0.382925724637681, "n": 368, "nrps": 0.13575035733208884, "score": 73.17734383695722, "top_ece": 0.20035138296438337 } }, "score": 70.7561729087212 }, "capability": 50.0091761607623, "composite_ci95": { "A": [ 6.289030902475565, 16.488978582767544 ], "B": [ 5.395378045720429, 15.122131514496889 ], "C": [ 4.367382571163586, 11.45067957136635 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.09682374384236453 }, "display": "kev 8B (research preview)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 28.412661059008222, "I_sealed": 30.111697766598596, "base": 29.26217941280341, "excess": -4.272424350028617, "gap": -1.6990367075903734, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 53.56342531883604, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864865, "hard": 43.378420887071414, "judge": 48.58272907053395, "standard": 69.8618668899121 } }, "open|noul": { "cc": 15.995169082125601, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 13.043478260869556, "judge": 10.000000000000009, "standard": -11.111111111111116 } }, "open|score": { "cc": 15.679388776063027, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 43.66574839302112, "hard": 33.21276294296097, "judge": 29.41280568734839, "standard": -53.98066473313996 } }, "sealed|choice": { "cc": 48.755934054679, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 61.4219200354741, "hard": 41.96832579185519, "judge": 49.72067039106147, "standard": 54.55105308535534 } }, "sealed|noul": { "cc": 19.06361274442773, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 43.99999999999999, "hard": -18.446601941747577, "judge": 50.0, "standard": 35.2112676056338 } }, "sealed|score": { "cc": 22.51554650068906, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 58.4626547761159, "hard": 14.194388556502712, "judge": 23.600291166841224, "standard": 19.557191252120088 } } } }, "jevbench_score": 10.62283914209564, "key": "kev-8b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 50, "ranked": true, "ranks": { "A": 50, "B": 50, "C": 50, "capability": 50 }, "scores": { "A": 10.62283914209564, "B": 9.44790466894511, "C": 7.3769716264553065 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2989678788930178, "p50_s_raw": 0.07448393944650888, "p95_s_adjusted": 0.42386933271773153, "p95_s_raw": 0.13693466635886578 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 65.36869404076147, 74.24437950494625 ], "I": [ 23.662522876918093, 35.13692561855455 ], "capability": [ 45.238684166857645, 53.83400001933101 ], "composite": { "A": [ 6.289030902475565, 16.488978582767544 ], "B": [ 5.395378045720429, 15.122131514496889 ], "C": [ 4.367382571163586, 11.45067957136635 ] }, "seed": 55 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 70.7561729087212, "capability": 50.0091761607623, "intelligence": 29.26217941280341 }, "per_type": { "choice": { "accuracy": 0.612, "calibration": 71.49009672209048, "cc": 50.38034159696103, "cc_by_type": { "choice": 50.38034159696103 }, "cc_tiered": 51.15967968675752, "cc_tiered_open": 53.56342531883604, "cc_tiered_sealed": 48.755934054679, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 50.38034159696103 }, "noul": { "accuracy": 0.6106666666666667, "calibration": 67.60107816711592, "cc": 22.13333333333334, "cc_by_type": { "noul": 22.13333333333334 }, "cc_tiered": 17.529390913276664, "cc_tiered_open": 15.995169082125601, "cc_tiered_sealed": 19.06361274442773, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 22.13333333333334 }, "score": { "accuracy": 0.49866666666666665, "calibration": 73.17734383695722, "cc": 23.802092840863022, "cc_by_type": { "score": 23.802092840863022 }, "cc_tiered": 19.097467638376042, "cc_tiered_open": 15.679388776063027, "cc_tiered_sealed": 22.51554650068906, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 23.802092840863022 } }, "raw_on_A": { "calibration": 67.24332350856244, "capability": 50.76538650167667, "intelligence": 34.2874494947909 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01866666666666672, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.01866666666666672, "n": 375 } }, "views": { "Balanced 33:33:33": 9.572183421009896, "Emphasis on Accuracy 60:20:20": 8.080814281596565, "Emphasis on Cost 20:20:60": 9.355272676593998, "Emphasis on Speed 20:60:20": 12.08216537216834, "Intelligence only": 6.550045158904166 }, "class": "jev-rebuild", "open": true, "licence": "Apache-2.0", "repo": "https://github.com/jaredpalmer/kev", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen3-8B-Base + LoRA + learned pointer head; jaredpalmer/kev-8b", "adapter_id": "images/kev-8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7654986522911051, "acc_among_decisive": 0.8063380281690141, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "FlyMy.AI (@denti)", "axes": { "calibration": 86.77033341356835, "cost": 66.37275756963365, "intelligence": 22.875473256337628, "speed": 90.39462801231525 }, "calibration": { "parts": { "choice": { "ece_hard": 0.060081291070015405, "ece_score": 87.98374178599691, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 87.98374178599691 }, "noul": { "abstention_rate": 0.6091644204851752, "brier": 0.15626137202130647, "ece": 0.06929060503851944, "n": 371, "score": 86.1418789922961 }, "score": { "interval90_coverage": 0.9539295392953929, "interval90_width": 0.646115627822945, "n": 369, "nrps": 0.13027713115375125, "score": 86.18537946241204, "top_ece": 0.07300763979900397 } }, "score": 86.77033341356835 }, "capability": 54.82290333495299, "composite_ci95": { "A": [ 6.070784931981477, 14.301728447276645 ], "B": [ 4.787007542130122, 11.888762823793376 ], "C": [ 4.215822869431582, 9.93175586616434 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.013210160098522166 }, "display": "Decision 2B (FlyMy.AI, v59)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 24.479797267586093, "I_sealed": 21.271149245089163, "base": 22.875473256337628, "excess": 0.6352603800586869, "gap": 3.2086480224969307, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 50.388518138122464, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 61.08108108108109, "hard": 45.26580685750237, "judge": 40.672379696769944, "standard": 69.8618668899121 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 14.28571428571428, "hard": -56.52173913043479, "judge": -19.999999999999996, "standard": 0.0 } }, "open|score": { "cc": 23.050873664635812, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 20.73971356294858, "hard": 15.558133148961872, "judge": 38.430985047283826, "standard": 16.12176767285528 } }, "sealed|choice": { "cc": 45.220796675917924, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 60.0916414160077, "hard": 37.38687782805428, "judge": 42.271880819366864, "standard": 57.876585786426915 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -4.0000000000000036, "hard": -53.398058252427184, "judge": -23.684210526315784, "standard": -18.309859154929576 } }, "sealed|score": { "cc": 18.59265105934957, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 27.03021479008173, "hard": 11.986008435765372, "judge": 25.699675889534046, "standard": 16.92661719587518 } } } }, "jevbench_score": 10.289708493744515, "key": "decision-2b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 51, "ranked": true, "ranks": { "A": 51, "B": 53, "C": 51, "capability": 48 }, "scores": { "A": 10.289708493744515, "B": 8.367001374285444, "C": 7.14563089843369 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.30028566285036506, "p50_s_raw": 0.07514283142518252, "p95_s_adjusted": 0.30409032410243525, "p95_s_raw": 0.07704516205121763 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 80.98496799808474, 88.16729740402266 ], "I": [ 18.649938213615208, 26.096089404676093 ], "capability": [ 50.712178970237844, 56.11601608037113 ], "composite": { "A": [ 6.070784931981477, 14.301728447276645 ], "B": [ 4.787007542130122, 11.888762823793376 ], "C": [ 4.215822869431582, 9.93175586616434 ] }, "seed": 29 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 86.77033341356835, "capability": 54.82290333495299, "intelligence": 22.875473256337628 }, "per_type": { "choice": { "accuracy": 0.5853333333333334, "calibration": 87.98374178599691, "cc": 46.9700557960649, "cc_by_type": { "choice": 46.9700557960649 }, "cc_tiered": 47.804657407020194, "cc_tiered_open": 50.388518138122464, "cc_tiered_sealed": 45.220796675917924, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 46.9700557960649 }, "noul": { "accuracy": 0.3626666666666667, "calibration": 86.1418789922961, "cc": -27.46666666666666, "cc_by_type": { "noul": -27.46666666666666 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.4266666666666667, "calibration": 86.18537946241204, "cc": 18.767847334171428, "cc_by_type": { "score": 18.767847334171428 }, "cc_tiered": 20.82176236199269, "cc_tiered_open": 23.050873664635812, "cc_tiered_sealed": 18.59265105934957, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 18.767847334171428 } }, "raw_on_A": { "calibration": 80.16146678308505, "capability": 51.30498083159781, "intelligence": 22.448494880110573 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 8.990679348778704, "Emphasis on Accuracy 60:20:20": 6.6544725899582415, "Emphasis on Cost 20:20:60": 10.468172682915556, "Emphasis on Speed 20:60:20": 11.379622803195113, "Intelligence only": 4.788177641426015 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 notices on the included code and the pinned base; the weights are an evaluation preview under EVALUATION-PERMISSION.md, not a cleared commercial release", "repo": "https://huggingface.co/flymy-ai/decision-2b-preview", "gpu": "RTX6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container", "underlying": "openbmb/MiniCPM5-2B with a trained LoRA adapter and pointer head (26.2M trainable parameters)", "adapter_id": "images/decision-2b/spec.json:flymy_v15.FlyMyPackage", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-05", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.3908355795148248, "acc_among_decisive": 0.9379310344827586, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "sabeel111", "axes": { "calibration": 73.92484912653177, "cost": 69.13657649738097, "intelligence": 23.217995107845883, "speed": 77.06362388237805 }, "calibration": { "parts": { "choice": { "ece_hard": 0.20911848208469058, "ece_score": 58.17630358306188, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 58.17630358306188 }, "noul": { "abstention_rate": 0.5121293800539084, "brier": 0.1552006734049543, "ece": 0.11625450134770884, "n": 371, "score": 76.74909973045823 }, "score": { "interval90_coverage": 0.9566395663956639, "interval90_width": 0.5600722673893404, "n": 369, "nrps": 0.11625319455925476, "score": 86.84914406607521, "top_ece": 0.07338196205962055 } }, "score": 73.92484912653177 }, "capability": 48.571422117188824, "composite_ci95": { "A": [ 6.483328619758367, 14.579365175086792 ], "B": [ 5.195391797769789, 12.338152010991331 ], "C": [ 4.502311541498866, 10.124559149365826 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0106851539408867 }, "display": "OpenSourceJev (Qwen3.5-4B Q4_K_M, native llama.cpp)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 24.81038701116274, "I_sealed": 21.62560320452903, "base": 23.217995107845883, "excess": 0.6113961641954666, "gap": 3.1847838066337104, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 41.32946193915506, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675674, "hard": 24.50456118276188, "judge": 64.40342781806196, "standard": 24.654667224780237 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": -4.347826086956519, "judge": -30.000000000000004, "standard": -22.22222222222222 } }, "open|score": { "cc": 33.101699094333156, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 67.34023030303031, "hard": 17.52584074074074, "judge": 47.55760169491523, "standard": 25.450296296296294 } }, "sealed|choice": { "cc": 35.66176229147756, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 48.11913384081, "hard": 23.133484162895915, "judge": 42.271880819366864, "standard": 44.57445498214066 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 24.0, "hard": -30.097087378640776, "judge": -7.8947368421052655, "standard": -4.225352112676061 } }, "sealed|score": { "cc": 29.21504732210953, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 58.979268794326224, "hard": 18.989292892156882, "judge": 31.412349700598806, "standard": 31.488491878172564 } } } }, "jevbench_score": 10.263493713193125, "key": "opensourcejev-qwen35-4b-q4km", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 52, "ranked": true, "ranks": { "A": 52, "B": 51, "C": 52, "capability": 53 }, "scores": { "A": 10.263493713193125, "B": 8.482178412964995, "C": 7.127426189717446 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.7395113246515393, "p50_s_raw": 0.29475566232576966, "p95_s_adjusted": 2.6588435446377843, "p95_s_raw": 1.2544217723188922 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 68.41904601995815, 77.70943106496745 ], "I": [ 19.379013533491335, 26.692735209928347 ], "capability": [ 44.976218992193104, 51.58836189201757 ], "composite": { "A": [ 6.483328619758367, 14.579365175086792 ], "B": [ 5.195391797769789, 12.338152010991331 ], "C": [ 4.502311541498866, 10.124559149365826 ] }, "seed": 84 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 73.92484912653177, "capability": 48.571422117188824, "intelligence": 23.217995107845883 }, "per_type": { "choice": { "accuracy": 0.5093333333333333, "calibration": 58.17630358306188, "cc": 37.25074126351086, "cc_by_type": { "choice": 37.25074126351086 }, "cc_tiered": 38.495612115316305, "cc_tiered_open": 41.32946193915506, "cc_tiered_sealed": 35.66176229147756, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 37.25074126351086 }, "noul": { "accuracy": 0.45866666666666667, "calibration": 76.74909973045823, "cc": -8.266666666666666, "cc_by_type": { "noul": -8.266666666666666 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.544, "calibration": 86.84914406607521, "cc": 30.80778824561413, "cc_by_type": { "score": 30.80778824561413 }, "cc_tiered": 31.15837320822134, "cc_tiered_open": 33.101699094333156, "cc_tiered_sealed": 29.21504732210953, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 30.80778824561413 } }, "raw_on_A": { "calibration": 73.9791977813484, "capability": 49.500913389632025, "intelligence": 25.022628997915646 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 9.174389111825883, "Emphasis on Accuracy 60:20:20": 6.882520870049428, "Emphasis on Cost 20:20:60": 10.842363781630336, "Emphasis on Speed 20:60:20": 11.176826098501683, "Intelligence only": 5.0064990418048145 }, "class": "jev-rebuild", "open": "unknown", "licence": "MIT (repository code); Apache-2.0 (Qwen/Qwen3.5-4B base and unsloth/Qwen3.5-4B-GGUF Q4_K_M conversion)", "repo": "https://github.com/sabeel111/OpenSourceJev", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": null, "adapter_id": "images/opensourcejev-qwen35-4b-q4km/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.48787061994609165, "acc_among_decisive": 0.9502762430939227, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "IkerMoel", "axes": { "calibration": 75.94384178347806, "cost": 63.27859776193043, "intelligence": 22.34610151162883, "speed": 91.30216983336405 }, "calibration": { "parts": { "choice": { "ece_hard": 0.15207997234343346, "ece_score": 69.58400553131331, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 69.58400553131331 }, "noul": { "abstention_rate": 0.4, "brier": 0.1703546052656481, "ece": 0.0957881739676287, "n": 375, "score": 80.84236520647426 }, "score": { "interval90_coverage": 0.8933333333333333, "interval90_width": 0.5735555555555554, "n": 375, "nrps": 0.14203249622577255, "score": 77.40515461264664, "top_ece": 0.1549322057606473 } }, "score": 75.94384178347806 }, "capability": 49.14497164755345, "composite_ci95": { "A": [ 6.323116186163818, 15.819609663846963 ], "B": [ 5.049803290865915, 13.396655191684498 ], "C": [ 4.391052907058209, 10.985840044338168 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (proxy tokens, I-2): exact base-model market reference; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0167512315270936 }, "display": "open-alternative-jev (Qwen3.5-4B, IkerMoel)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 25.032294599047653, "I_sealed": 19.659908424210006, "base": 22.34610151162883, "excess": 2.7989985323994033, "gap": 5.372386174837647, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 51.43479500294909, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 64.97297297297297, "hard": 43.378420887071414, "judge": 60.44825313117996, "standard": 47.25826705734617 } }, "open|noul": { "cc": 0.8184955141476893, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 28.57142857142858, "hard": 4.347826086956519, "judge": -19.999999999999996, "standard": 11.111111111111116 } }, "open|score": { "cc": 22.843593280046196, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 40.9867728497478, "hard": 12.570359188934388, "judge": 28.72169648259133, "standard": 25.50131687360131 } }, "sealed|choice": { "cc": 44.97077692151599, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 57.43108417707487, "hard": 39.93212669683257, "judge": 40.409683426443216, "standard": 55.65956398571254 } }, "sealed|noul": { "cc": 0.6994422333695622, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 32.00000000000001, "hard": -14.563106796116509, "judge": 2.6315789473684292, "standard": 12.676056338028175 } }, "sealed|score": { "cc": 13.309506117744464, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 44.90364228457862, "hard": 10.653374925310422, "judge": 6.439637126414932, "standard": 13.129503906189765 } } } }, "jevbench_score": 9.435692571873393, "key": "open-alternative-jev", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 53, "ranked": true, "ranks": { "A": 53, "B": 54, "C": 53, "capability": 52 }, "scores": { "A": 9.435692571873393, "B": 7.716442780206986, "C": 6.5525642860231885 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2366679080063477, "p50_s_raw": 0.04333395400317386, "p95_s_adjusted": 0.3130715797946323, "p95_s_raw": 0.08153578989731614 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 70.67388183281524, 78.71286748398254 ], "I": [ 19.129525848760927, 27.488222507229114 ], "capability": [ 45.43710032043718, 52.3768690725618 ], "composite": { "A": [ 6.323116186163818, 15.819609663846963 ], "B": [ 5.049803290865915, 13.396655191684498 ], "C": [ 4.391052907058209, 10.985840044338168 ] }, "seed": 74 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 75.94384178347806, "capability": 49.14497164755345, "intelligence": 22.34610151162883 }, "per_type": { "choice": { "accuracy": 0.5866666666666667, "calibration": 69.58400553131331, "cc": 47.14057008610969, "cc_by_type": { "choice": 47.14057008610969 }, "cc_tiered": 48.20278596223254, "cc_tiered_open": 51.43479500294909, "cc_tiered_sealed": 44.97077692151599, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 47.14057008610969 }, "noul": { "accuracy": 0.52, "calibration": 80.84236520647426, "cc": 4.0000000000000036, "cc_by_type": { "noul": 4.0000000000000036 }, "cc_tiered": 0.7589688737586258, "cc_tiered_open": 0.8184955141476893, "cc_tiered_sealed": 0.6994422333695622, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 4.0000000000000036 }, "score": { "accuracy": 0.4106666666666667, "calibration": 77.40515461264664, "cc": 17.046666794552323, "cc_by_type": { "score": 17.046666794552323 }, "cc_tiered": 18.07654969889533, "cc_tiered_open": 22.843593280046196, "cc_tiered_sealed": 13.309506117744464, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 17.046666794552323 } }, "raw_on_A": { "calibration": 76.45819741974654, "capability": 51.99334325152506, "intelligence": 27.528489083303576 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 8.37993057455771, "Emphasis on Accuracy 60:20:20": 6.202799597209423, "Emphasis on Cost 20:20:60": 9.685497012220603, "Emphasis on Speed 20:60:20": 10.691355012814217, "Intelligence only": 4.463394698403469 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 (code and weights)", "repo": "https://github.com/ikermoel/open-alternative-jev", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": "Qwen/Qwen3.5-4B (frozen, BF16)", "adapter_id": "images/open-alternative-jev/spec.json:remote_inproc_v15.RemoteInprocV15Adapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6, "acc_among_decisive": 0.8666666666666667, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Alibaba Qwen / neutral reproduction", "axes": { "calibration": 33.721548452619835, "cost": 45.53515592313027, "intelligence": 26.54912920590792, "speed": 88.65282990878164 }, "calibration": { "parts": { "choice": { "ece_hard": 0.43645983930453747, "ece_score": 12.708032139092506, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 12.708032139092506 }, "noul": { "abstention_rate": 0.07816711590296499, "brier": 0.2740805017981965, "ece": 0.2700267098593338, "n": 371, "score": 45.99465802813324 }, "score": { "interval90_coverage": 0.5474254742547425, "interval90_width": 0.14972899728997294, "n": 369, "nrps": 0.21398318753780107, "score": 42.461955190633766, "top_ece": 0.46838885432476185 } }, "score": 33.721548452619835 }, "capability": 30.13533882926388, "composite_ci95": { "A": [ 5.526896473528715, 13.605697544750722 ], "B": [ 4.901937683715855, 12.59857128804635 ], "C": [ 3.838122551061608, 9.448401072743554 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0653875104679803 }, "display": "Raw Qwen3 8B direct logits", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 31.420181182575643, "I_sealed": 21.6780772292402, "base": 26.54912920590792, "excess": 7.168716310897201, "gap": 9.742103953335445, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 51.791553839411165, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 61.08108108108109, "hard": 33.941491034916645, "judge": 60.44825313117996, "standard": 69.8618668899121 } }, "open|noul": { "cc": 34.967563837129056, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 30.434782608695656, "judge": 39.99999999999999, "standard": 11.111111111111116 } }, "open|score": { "cc": 7.50142587118671, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 42.05293172396295, "hard": 3.1607638000191285, "judge": 1.3576889332232334, "standard": 8.122602494078969 } }, "sealed|choice": { "cc": 41.65523094061268, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 57.43108417707487, "hard": 31.787330316742068, "judge": 39.47858472998139, "standard": 56.76807488606972 } }, "sealed|noul": { "cc": 22.61439253816204, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 60.00000000000001, "hard": -6.796116504854366, "judge": 31.578947368421062, "standard": 49.29577464788733 } }, "sealed|score": { "cc": 0.7646082089458659, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 50.377225661492794, "hard": -19.926430375972124, "judge": 5.951813095928216, "standard": 9.559569322034855 } } } }, "jevbench_score": 9.30125972789427, "key": "raw-qwen3-8b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 54, "ranked": true, "ranks": { "A": 54, "B": 52, "C": 54, "capability": 84 }, "scores": { "A": 9.30125972789427, "B": 8.458419096926727, "C": 6.4592081443710185 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.30506142796948554, "p50_s_raw": 0.07753071398474276, "p95_s_adjusted": 0.4470228368416429, "p95_s_raw": 0.14851141842082144 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 28.041560709983848, 38.19473887695128 ], "I": [ 21.71577559015635, 30.903657696346237 ], "capability": [ 25.49211567892071, 33.98867030681561 ], "composite": { "A": [ 5.526896473528715, 13.605697544750722 ], "B": [ 4.901937683715855, 12.59857128804635 ], "C": [ 3.838122551061608, 9.448401072743554 ] }, "seed": 97 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 33.721548452619835, "capability": 30.13533882926388, "intelligence": 26.54912920590792 }, "per_type": { "choice": { "accuracy": 0.5613333333333334, "calibration": 12.708032139092506, "cc": 43.90079857525836, "cc_by_type": { "choice": 43.90079857525836 }, "cc_tiered": 46.723392390011924, "cc_tiered_open": 51.791553839411165, "cc_tiered_sealed": 41.65523094061268, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 43.90079857525836 }, "noul": { "accuracy": 0.648, "calibration": 45.99465802813324, "cc": 29.600000000000005, "cc_by_type": { "noul": 29.600000000000005 }, "cc_tiered": 28.79097818764555, "cc_tiered_open": 34.967563837129056, "cc_tiered_sealed": 22.61439253816204, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 29.600000000000005 }, "score": { "accuracy": 0.3973333333333333, "calibration": 42.461955190633766, "cc": 3.4025581754630996, "cc_by_type": { "score": 3.4025581754630996 }, "cc_tiered": 4.133017040066288, "cc_tiered_open": 7.50142587118671, "cc_tiered_sealed": 0.7646082089458659, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 3.4025581754630996 } }, "raw_on_A": { "calibration": 38.60246474366818, "capability": 35.16687873306722, "intelligence": 31.73129272246627 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 9.893413746086136, "Emphasis on Accuracy 60:20:20": 7.995042950818957, "Emphasis on Cost 20:20:60": 10.181976971451938, "Emphasis on Speed 20:60:20": 12.509103386849494, "Intelligence only": 6.208182595906101 }, "class": "raw-logit-control", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/Qwen/Qwen3-8B", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen3-8B BF16", "adapter_id": "images/raw-qwen3-8b/spec.json:raw_adapter.RawControlAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.921832884097035, "acc_among_decisive": 0.7105263157894737, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Mapika", "axes": { "calibration": 68.52438190259257, "cost": 64.91766177740826, "intelligence": 20.654576369710338, "speed": 94.55945724666857 }, "calibration": { "parts": { "choice": { "ece_hard": 0.18752718750000008, "ece_score": 62.49456249999999, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 62.49456249999999 }, "noul": { "abstention_rate": 0.31999999999999995, "brier": 0.21768861922666663, "ece": 0.1706301333333334, "n": 375, "score": 65.87397333333331 }, "score": { "interval90_coverage": 0.8746666666666667, "interval90_width": 0.5113333333333334, "n": 375, "nrps": 0.14404540251111114, "score": 77.20460987444443, "top_ece": 0.1559312000000001 } }, "score": 68.52438190259257 }, "capability": 44.58947913615145, "composite_ci95": { "A": [ 4.470553979976864, 13.010288397584645 ], "B": [ 3.512028417919115, 10.890754224642697 ], "C": [ 3.1045513749839326, 9.034922498322668 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.014771046798029557 }, "display": "decider-2b (Mapika)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 22.803908585466385, "I_sealed": 18.50524415395429, "base": 20.654576369710338, "excess": 1.7252767890738503, "gap": 4.298664431512094, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 37.01745599452806, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 61.08108108108109, "hard": 24.50456118276188, "judge": 28.80685563612393, "standard": 62.32733361239012 } }, "open|noul": { "cc": 4.469289164941341, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 42.85714285714286, "hard": 4.347826086956519, "judge": -19.999999999999996, "standard": 22.222222222222232 } }, "open|score": { "cc": 26.92498059692975, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 60.77454545454546, "hard": 25.165648148148158, "judge": 23.80372881355931, "standard": 18.20074074074074 } }, "sealed|choice": { "cc": 39.65284081929382, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 58.7613627965413, "hard": 29.24208144796379, "judge": 35.7541899441341, "standard": 56.76807488606972 } }, "sealed|noul": { "cc": 0.12111092718806482, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 39.99999999999999, "hard": -28.155339805825243, "judge": 10.526315789473696, "standard": 21.126760563380277 } }, "sealed|score": { "cc": 15.741780715380987, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 52.834113475177304, "hard": 4.744583333333363, "judge": 10.707844311377212, "standard": 26.740913705583736 } } } }, "jevbench_score": 7.6704332123718, "key": "decider-2b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 55, "ranked": true, "ranks": { "A": 55, "B": 55, "C": 55, "capability": 58 }, "scores": { "A": 7.6704332123718, "B": 6.209608793546842, "C": 5.326689730813751 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.17949020988307893, "p50_s_raw": 0.014745104941539466, "p95_s_adjusted": 0.1949905256042257, "p95_s_raw": 0.022495262802112847 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 62.32738945440369, 71.2629933305287 ], "I": [ 16.778817874747006, 25.441954749747556 ], "capability": [ 40.31419112262012, 47.6366012348925 ], "composite": { "A": [ 4.470553979976864, 13.010288397584645 ], "B": [ 3.512028417919115, 10.890754224642697 ], "C": [ 3.1045513749839326, 9.034922498322668 ] }, "seed": 26 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 68.52438190259257, "capability": 44.58947913615145, "intelligence": 20.654576369710338 }, "per_type": { "choice": { "accuracy": 0.532, "calibration": 62.49456249999999, "cc": 40.149484194272596, "cc_by_type": { "choice": 40.149484194272596 }, "cc_tiered": 38.335148406910946, "cc_tiered_open": 37.01745599452806, "cc_tiered_sealed": 39.65284081929382, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 40.149484194272596 }, "noul": { "accuracy": 0.528, "calibration": 65.87397333333331, "cc": 5.600000000000005, "cc_by_type": { "noul": 5.600000000000005 }, "cc_tiered": 2.295200046064703, "cc_tiered_open": 4.469289164941341, "cc_tiered_sealed": 0.12111092718806482, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 5.600000000000005 }, "score": { "accuracy": 0.41333333333333333, "calibration": 77.20460987444443, "cc": 20.228824561403492, "cc_by_type": { "score": 20.228824561403492 }, "cc_tiered": 21.33338065615537, "cc_tiered_open": 26.92498059692975, "cc_tiered_sealed": 15.741780715380987, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 20.228824561403492 } }, "raw_on_A": { "calibration": 62.755705376823364, "capability": 43.8983266486265, "intelligence": 25.040947920429637 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 6.881302219979439, "Emphasis on Accuracy 60:20:20": 4.98303010129531, "Emphasis on Cost 20:20:60": 8.110241448995462, "Emphasis on Speed 20:60:20": 8.93000700376026, "Intelligence only": 3.5245921294252027 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/Mapika/decider-2b", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "Qwen3.5-2B-Base with a trained decision readout, 1.9B", "adapter_id": "images/decider-2b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.68, "acc_among_decisive": 0.7764705882352941, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "pst2154", "axes": { "calibration": 74.53740737677316, "cost": 55.49170925809556, "intelligence": 20.38848344571935, "speed": 93.8305724848098 }, "calibration": { "parts": { "choice": { "ece_hard": 0.10832159242229476, "ece_score": 78.33568151554104, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 78.33568151554104 }, "noul": { "abstention_rate": 0.3881401617250674, "brier": 0.22241599402070583, "ece": 0.17236678386479326, "n": 371, "score": 65.52664322704135 }, "score": { "interval90_coverage": 0.9214092140921409, "interval90_width": 0.6684733514001803, "n": 369, "nrps": 0.14545408533817059, "score": 79.74989738773714, "top_ece": 0.12977398345354327 } }, "score": 74.53740737677316 }, "capability": 47.46294541124625, "composite_ci95": { "A": [ 4.1676910325965215, 10.20449077801364 ], "B": [ 3.2802107600784733, 8.482218509265811 ], "C": [ 2.894229883747585, 7.086451929176138 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.03045157019704434 }, "display": "Nemotron Diffusion 8B (pst2154, optimized vLLM)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 22.973528132452522, "I_sealed": 17.803438758986175, "base": 20.38848344571935, "excess": 2.596701731028103, "gap": 5.170089373466347, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 45.141807097909854, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 37.72972972972973, "hard": 32.05410506448569, "judge": 48.58272907053395, "standard": 69.8618668899121 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": -21.739130434782606, "judge": -40.0, "standard": -11.111111111111116 } }, "open|score": { "cc": 23.77877729944772, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 44.848253383055216, "hard": 4.46243778457357, "judge": 33.03604774332046, "standard": 37.99081262158317 } }, "sealed|choice": { "cc": 40.98559356961426, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 61.4219200354741, "hard": 25.169683257918543, "judge": 50.65176908752329, "standard": 47.89998768321223 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 43.99999999999999, "hard": -43.689320388349515, "judge": 5.263157894736836, "standard": -1.4084507042253502 } }, "sealed|score": { "cc": 12.424722707344273, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 42.099894800900906, "hard": 9.652238984201755, "judge": 2.7258103611165985, "standard": 17.6804726261925 } } } }, "jevbench_score": 7.29748476516348, "key": "nemotron-diffusion-8b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 56, "ranked": true, "ranks": { "A": 56, "B": 56, "C": 56, "capability": 56 }, "scores": { "A": 7.29748476516348, "B": 5.930430684235979, "C": 5.06769775358575 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.1939013559371233, "p50_s_raw": 0.02195067796856165, "p95_s_adjusted": 0.21348231588490307, "p95_s_raw": 0.03174115794245154 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 68.2662042332654, 77.36530331396067 ], "I": [ 16.421783930074305, 23.311693043206645 ], "capability": [ 43.39466614794243, 49.39379257235283 ], "composite": { "A": [ 4.1676910325965215, 10.20449077801364 ], "B": [ 3.2802107600784733, 8.482218509265811 ], "C": [ 2.894229883747585, 7.086451929176138 ] }, "seed": 70 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 74.53740737677316, "capability": 47.46294541124625, "intelligence": 20.38848344571935 }, "per_type": { "choice": { "accuracy": 0.54, "calibration": 78.33568151554104, "cc": 41.17256993454144, "cc_by_type": { "choice": 41.17256993454144 }, "cc_tiered": 43.06370033376206, "cc_tiered_open": 45.141807097909854, "cc_tiered_sealed": 40.98559356961426, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 41.17256993454144 }, "noul": { "accuracy": 0.464, "calibration": 65.52664322704135, "cc": -7.199999999999996, "cc_by_type": { "noul": -7.199999999999996 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.4266666666666667, "calibration": 79.74989738773714, "cc": 16.44673378084317, "cc_by_type": { "score": 16.44673378084317 }, "cc_tiered": 18.101750003396, "cc_tiered_open": 23.77877729944772, "cc_tiered_sealed": 12.424722707344273, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 16.44673378084317 } }, "raw_on_A": { "calibration": 75.92799396845535, "capability": 48.766079474284126, "intelligence": 21.6041649801129 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 6.417819358792639, "Emphasis on Accuracy 60:20:20": 4.7285860819837495, "Emphasis on Cost 20:20:60": 7.307751777390142, "Emphasis on Speed 20:60:20": 8.394346515316924, "Intelligence only": 3.3901175711211273 }, "class": "system-one-open", "open": "yes", "licence": "MIT (application code); modified vLLM runtime Apache-2.0; nvidia/Nemotron-Labs-Diffusion-8B keeps its upstream licence", "repo": "https://github.com/pst2154/Nemotron_Jev", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "nvidia/Nemotron-Labs-Diffusion-8B @ 16c67f05 (unchanged weights) served by github.com/pst2154/Nemotron_Jev @ cdc36d60 on the author's digest-pinned modified vLLM runtime: BF16, candidate-only output projection, direct candidate logits, CUDA graphs, one masked forward pass per decision, nothing generated. Disclosed by the author: the inherited option-ordering rule was selected using public benchmark examples in earlier 14B experiments. Our earlier v1.4.3 run of this system used a recipe that left the base image's older 14B application in place; this v1.5 row is a fresh measurement of the 8B.", "adapter_id": "a3/candidates/nemotron-diffusion-8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6118598382749326, "acc_among_decisive": 0.7665198237885462, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Jared Palmer", "axes": { "calibration": 68.45285936403559, "cost": 65.78036497095258, "intelligence": 19.510707720813777, "speed": 90.46585183268016 }, "calibration": { "parts": { "choice": { "ece_hard": 0.18711800351703844, "ece_score": 62.576399296592314, "mean_tvd": null, "n": 733, "n_ece_hard": 306, "n_tvd": 0, "score": 62.576399296592314 }, "noul": { "abstention_rate": 0.32884097035040427, "brier": 0.19162911051212952, "ece": 0.13239892183288407, "n": 371, "score": 73.52021563342319 }, "score": { "interval90_coverage": 0.8365122615803815, "interval90_width": 0.4604904632152589, "n": 367, "nrps": 0.15784808091493369, "score": 69.26196316209126, "top_ece": 0.22845632792162052 } }, "score": 68.45285936403559 }, "capability": 43.98178354242468, "composite_ci95": { "A": [ 4.075524448580174, 12.02725721176505 ], "B": [ 3.1848561369918413, 10.0344884794078 ], "C": [ 2.83022531151401, 8.352261952614619 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.013824661330049262 }, "display": "kev 4B (research preview)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 19.939346418532395, "I_sealed": 19.08206902309516, "base": 19.510707720813777, "excess": -1.7161102470010072, "gap": 0.8572773954372366, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 51.235731456250235, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675674, "hard": 45.26580685750237, "judge": 44.62755438365195, "standard": 62.32733361239012 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": -4.347826086956519, "judge": -19.999999999999996, "standard": -11.111111111111116 } }, "open|score": { "cc": 8.582307799346953, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 34.24426078971534, "hard": 15.014273649587162, "judge": 21.314617902468203, "standard": -36.211065550999535 } }, "sealed|choice": { "cc": 42.547775845492595, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 56.10080555760846, "hard": 29.751131221719444, "judge": 44.134078212290504, "standard": 58.9850966867841 } }, "sealed|noul": { "cc": 5.208014566705289, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 39.99999999999999, "hard": -30.097087378640776, "judge": 26.315789473684205, "standard": 26.7605633802817 } }, "sealed|score": { "cc": 9.490416657087591, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 60.8431474353109, "hard": -5.464015519199061, "judge": 4.345254884769856, "standard": 21.440658279025847 } } } }, "jevbench_score": 6.611791819903415, "key": "kev-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 57, "ranked": true, "ranks": { "A": 57, "B": 57, "C": 57, "capability": 61 }, "scores": { "A": 6.611791819903415, "B": 5.310194944940352, "C": 4.591522097155148 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.29399301540106537, "p50_s_raw": 0.07199650770053267, "p95_s_adjusted": 0.3055468452628701, "p95_s_raw": 0.07777342263143508 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 62.30814638480004, 71.54002800358278 ], "I": [ 16.228114704223106, 24.71673328753193 ], "capability": [ 39.998560456744215, 47.239203316922335 ], "composite": { "A": [ 4.075524448580174, 12.02725721176505 ], "B": [ 3.1848561369918413, 10.0344884794078 ], "C": [ 2.83022531151401, 8.352261952614619 ] }, "seed": 54 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 68.45285936403559, "capability": 43.98178354242468, "intelligence": 19.510707720813777 }, "per_type": { "choice": { "accuracy": 0.5693333333333334, "calibration": 62.576399296592314, "cc": 44.923884315527204, "cc_by_type": { "choice": 44.923884315527204 }, "cc_tiered": 46.891753650871415, "cc_tiered_open": 51.235731456250235, "cc_tiered_sealed": 42.547775845492595, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 44.923884315527204 }, "noul": { "accuracy": 0.5413333333333333, "calibration": 73.52021563342319, "cc": 8.266666666666666, "cc_by_type": { "noul": 8.266666666666666 }, "cc_tiered": 2.6040072833526446, "cc_tiered_open": 0.0, "cc_tiered_sealed": 5.208014566705289, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 8.266666666666666 }, "score": { "accuracy": 0.41333333333333333, "calibration": 69.26196316209126, "cc": 12.570612763030764, "cc_by_type": { "score": 12.570612763030764 }, "cc_tiered": 9.036362228217271, "cc_tiered_open": 8.582307799346953, "cc_tiered_sealed": 9.490416657087591, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 12.570612763030764 } }, "raw_on_A": { "calibration": 69.97372951951687, "capability": 47.498407356877294, "intelligence": 25.023085194237723 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.022666666666666613, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.021333333333333315, "n": 375 } }, "views": { "Balanced 33:33:33": 5.893456455385603, "Emphasis on Accuracy 60:20:20": 4.229224928104588, "Emphasis on Cost 20:20:60": 7.055012442959745, "Emphasis on Speed 20:60:20": 7.642572180991565, "Intelligence only": 2.970838616432075 }, "class": "jev-rebuild", "open": true, "licence": "Apache-2.0", "repo": "https://github.com/jaredpalmer/kev", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen3-4B-Base + LoRA + learned pointer head; jaredpalmer/kev-4b", "adapter_id": "images/kev-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6711590296495957, "acc_among_decisive": 0.8152610441767069, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "newfull5 (dhtocks)", "axes": { "calibration": 79.34885520197874, "cost": 65.5661191189412, "intelligence": 15.917395195670291, "speed": 92.03462875001455 }, "calibration": { "parts": { "choice": { "ece_hard": 0.1264719869706841, "ece_score": 74.70560260586319, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 74.70560260586319 }, "noul": { "abstention_rate": 0.46630727762803237, "brier": 0.1896760988948787, "ece": 0.09166172506738543, "n": 371, "score": 81.6676549865229 }, "score": { "interval90_coverage": 0.9024390243902439, "interval90_width": 0.6144986449864495, "n": 369, "nrps": 0.14966934108401095, "score": 81.67330801355013, "top_ece": 0.10843224932249321 } }, "score": 79.34885520197874 }, "capability": 47.633125198824516, "composite_ci95": { "A": [ 2.1896531510798423, 6.224018858922979 ], "B": [ 1.639660951009494, 4.9293547200832295 ], "C": [ 1.5205924660276677, 4.322235318696512 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.014053873152709357 }, "display": "Malkuth-2B (newfull5, Kev post-train)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 16.94824627575026, "I_sealed": 14.88654411559032, "base": 15.917395195670291, "excess": -0.511685482278303, "gap": 2.061702160159941, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 24.95619850881217, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 61.08108108108109, "hard": 16.955017301038072, "judge": 28.80685563612393, "standard": 17.120133947258267 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 28.57142857142858, "hard": -21.739130434782606, "judge": -40.0, "standard": 0.0 } }, "open|score": { "cc": 25.88854031843862, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 49.77636363636364, "hard": 12.745185185185182, "judge": 28.807457627118627, "standard": 35.852962962962955 } }, "sealed|choice": { "cc": 39.92521849097588, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 50.77969107974283, "hard": 26.18778280542985, "judge": 44.134078212290504, "standard": 55.65956398571254 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 28.000000000000004, "hard": -39.80582524271845, "judge": -23.684210526315784, "standard": 18.309859154929576 } }, "sealed|score": { "cc": 4.734413855795083, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 40.8195035460993, "hard": -11.993921568627375, "judge": 11.762814371257434, "standard": 9.605939086294368 } } } }, "jevbench_score": 3.992169201455145, "key": "malkuth-2b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 58, "ranked": true, "ranks": { "A": 58, "B": 59, "C": 58, "capability": 55 }, "scores": { "A": 3.992169201455145, "B": 3.08287077876474, "C": 2.7723397232327396 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.23610859100008383, "p50_s_raw": 0.04305429550004192, "p95_s_adjusted": 0.26510952737880866, "p95_s_raw": 0.05755476368940435 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 72.7053529716356, 81.31914491234029 ], "I": [ 12.692506481394203, 18.87608002604669 ], "capability": [ 43.451731383062054, 49.395128138793496 ], "composite": { "A": [ 2.1896531510798423, 6.224018858922979 ], "B": [ 1.639660951009494, 4.9293547200832295 ], "C": [ 1.5205924660276677, 4.322235318696512 ] }, "seed": 63 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 79.34885520197874, "capability": 47.633125198824516, "intelligence": 15.917395195670291 }, "per_type": { "choice": { "accuracy": 0.5133333333333333, "calibration": 74.70560260586319, "cc": 37.76228413364529, "cc_by_type": { "choice": 37.76228413364529 }, "cc_tiered": 32.44070849989403, "cc_tiered_open": 24.95619850881217, "cc_tiered_sealed": 39.92521849097588, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 37.76228413364529 }, "noul": { "accuracy": 0.44533333333333336, "calibration": 81.6676549865229, "cc": -10.933333333333328, "cc_by_type": { "noul": -10.933333333333328 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.38133333333333336, "calibration": 81.67330801355013, "cc": 9.127684210526233, "cc_by_type": { "score": 9.127684210526233 }, "cc_tiered": 15.31147708711685, "cc_tiered_open": 25.88854031843862, "cc_tiered_sealed": 4.734413855795083, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 9.127684210526233 } }, "raw_on_A": { "calibration": 76.45071634453062, "capability": 47.2825063700716, "intelligence": 18.114296395612577 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 3.4183795643165196, "Emphasis on Accuracy 60:20:20": 2.361369226233909, "Emphasis on Cost 20:20:60": 4.242339522263916, "Emphasis on Speed 20:60:20": 4.578614246342653, "Intelligence only": 1.6131545908775944 }, "class": "jev-rebuild", "open": "yes", "licence": "CC-BY-NC-4.0, research use only (XNLI and RACE in the training mix)", "repo": "https://github.com/newfull5/malkuth", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": null, "adapter_id": "images/malkuth-2b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5336927223719676, "acc_among_decisive": 0.8434343434343434, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Qwen", "axes": { "calibration": 73.15350658389404, "cost": 48.39770052432463, "intelligence": 16.337944014530347, "speed": 82.14971164581814 }, "calibration": { "parts": { "choice": { "ece_hard": 0.12818639012273808, "ece_score": 74.36272197545239, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 74.36272197545239 }, "noul": { "abstention_rate": 0.7466666666666666, "brier": 0.23186747724503004, "ece": 0.1932172851672485, "n": 375, "score": 61.3565429665503 }, "score": { "interval90_coverage": 0.9093333333333333, "interval90_width": 0.6708888888888882, "n": 375, "nrps": 0.15088039394905625, "score": 83.74125480967948, "top_ece": 0.08714725492867716 } }, "score": 73.15350658389404 }, "capability": 44.7457252992122, "composite_ci95": { "A": [ 2.2277188584602894, 5.423744955848025 ], "B": [ 1.7231876454208677, 4.422291990633802 ], "C": [ 1.5470269850418672, 3.766489552672239 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: hosted exact-model reference (deepinfra:Qwen/Qwen3-Reranker-4B); no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.05249000923645321 }, "display": "Qwen3-Reranker-4B", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 17.351796606180663, "I_sealed": 15.324091422880029, "base": 16.337944014530347, "excess": -0.5456824591376095, "gap": 2.0277051833006343, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 36.57581104234733, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 53.2972972972973, "hard": 22.617175212330924, "judge": 52.537903757415954, "standard": 32.18920050230222 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -57.14285714285714, "hard": -91.30434782608697, "judge": -40.0, "standard": -66.66666666666667 } }, "open|score": { "cc": 15.479578776194659, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 16.665491764012373, "hard": -1.4208863867574584, "judge": 33.97314797019466, "standard": 20.94719881719004 } }, "sealed|choice": { "cc": 33.418936417285686, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 45.458576601877176, "hard": 22.115384615384603, "judge": 34.823091247672274, "standard": 47.89998768321223 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -43.99999999999999, "hard": -84.46601941747574, "judge": -57.89473684210527, "standard": -43.66197183098591 } }, "sealed|score": { "cc": 12.553337851354403, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 16.14176889876734, "hard": 9.253841484765735, "judge": 15.752141635882, "standard": 12.559909384033885 } } } }, "jevbench_score": 3.715006798009075, "key": "qwen3-reranker-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 59, "ranked": true, "ranks": { "A": 59, "B": 60, "C": 59, "capability": 57 }, "scores": { "A": 3.715006798009075, "B": 2.961112195421367, "C": 2.5798658319507455 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.5581826242618263, "p50_s_raw": 0.20409131213091314, "p95_s_adjusted": 1.092075143335387, "p95_s_raw": 0.47103757166769356 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 67.92034254293053, 76.98495141643136 ], "I": [ 13.411885193349915, 18.933037790738602 ], "capability": [ 41.28670365010265, 47.260558123220996 ], "composite": { "A": [ 2.2277188584602894, 5.423744955848025 ], "B": [ 1.7231876454208677, 4.422291990633802 ], "C": [ 1.5470269850418672, 3.766489552672239 ] }, "seed": 91 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 73.15350658389404, "capability": 44.7457252992122, "intelligence": 16.337944014530347 }, "per_type": { "choice": { "accuracy": 0.4866666666666667, "calibration": 74.36272197545239, "cc": 34.35199833274914, "cc_by_type": { "choice": 34.35199833274914 }, "cc_tiered": 34.99737372981651, "cc_tiered_open": 36.57581104234733, "cc_tiered_sealed": 33.418936417285686, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 34.35199833274914 }, "noul": { "accuracy": 0.18933333333333333, "calibration": 61.3565429665503, "cc": -62.133333333333326, "cc_by_type": { "noul": -62.133333333333326 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.37333333333333335, "calibration": 83.74125480967948, "cc": 12.388563279917397, "cc_by_type": { "score": 12.388563279917397 }, "cc_tiered": 14.016458313774532, "cc_tiered_open": 15.479578776194659, "cc_tiered_sealed": 12.553337851354403, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 12.388563279917397 } }, "raw_on_A": { "calibration": 68.75797234609402, "capability": 42.734177704587836, "intelligence": 16.71038306308165 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 3.1912622823260945, "Emphasis on Accuracy 60:20:20": 2.310804869814862, "Emphasis on Cost 20:20:60": 3.695072544212985, "Emphasis on Speed 20:60:20": 4.224997058133206, "Intelligence only": 1.6344124017617998 }, "class": "reranker", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/Qwen/Qwen3-Reranker-4B", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "4B instruction-aware generative reranker", "adapter_id": "images/qwen3-reranker-4b/spec.json:reranker_adapter.RerankerAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.25333333333333335, "acc_among_decisive": 0.7473684210526316, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "ZeroEntropy", "axes": { "calibration": 83.20040692874471, "cost": 48.39770052432463, "intelligence": 15.596611891228248, "speed": 82.89783918855927 }, "calibration": { "parts": { "choice": { "ece_hard": 0.09594830772201698, "ece_score": 80.8103384555966, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 80.8103384555966 }, "noul": { "abstention_rate": 0.8613333333333333, "brier": 0.2189278600834461, "ece": 0.0929386859803417, "n": 375, "score": 81.41226280393165 }, "score": { "interval90_coverage": 0.984, "interval90_width": 0.9126666666666668, "n": 375, "nrps": 0.14987040001080157, "score": 87.3786195267059, "top_ece": 0.05127860472754006 } }, "score": 83.20040692874471 }, "capability": 49.39850940998648, "composite_ci95": { "A": [ 1.9752898202498443, 4.84420234615273 ], "B": [ 1.507128562340162, 3.903609846255529 ], "C": [ 1.3717290418401693, 3.364029407050507 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: size-class proxy (deepinfra:Qwen/Qwen3-Reranker-4B); no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.05249000923645321 }, "display": "ZeroEntropy zerank-2", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 15.496935616681888, "I_sealed": 15.696288165774607, "base": 15.596611891228248, "excess": -2.772740191530963, "gap": -0.19935254909271904, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 36.28749068653345, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 49.40540540540541, "hard": 30.16671909405474, "judge": 32.76203032300594, "standard": 47.25826705734617 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -100.0, "hard": -82.6086956521739, "judge": -70.0, "standard": -77.77777777777779 } }, "open|score": { "cc": 10.203316163512213, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 13.133808997075057, "hard": 3.6957598546269232, "judge": 27.616853766877792, "standard": -4.367124040546999 } }, "sealed|choice": { "cc": 34.99399091747191, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 50.77969107974283, "hard": 27.205882352941163, "judge": 36.68528864059592, "standard": 40.14041138071193 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -88.0, "hard": -86.40776699029126, "judge": -71.05263157894737, "standard": -52.112676056338024 } }, "sealed|score": { "cc": 12.094873579851914, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 16.780298372211313, "hard": 6.472246249261849, "judge": 17.43542452331028, "standard": 12.986589429664797 } } } }, "jevbench_score": 3.349795360318651, "key": "zerank-2", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 60, "ranked": true, "ranks": { "A": 60, "B": 62, "C": 60, "capability": 51 }, "scores": { "A": 3.349795360318651, "B": 2.635183024414276, "C": 2.3262467779990628 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.4991468541324139, "p50_s_raw": 0.17457342706620693, "p95_s_adjusted": 1.0279872846323996, "p95_s_raw": 0.4389936423161998 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 77.23040686190234, 86.00830164706042 ], "I": [ 12.746131677720898, 18.039638464222975 ], "capability": [ 46.43325156721317, 50.75210427056033 ], "composite": { "A": [ 1.9752898202498443, 4.84420234615273 ], "B": [ 1.507128562340162, 3.903609846255529 ], "C": [ 1.3717290418401693, 3.364029407050507 ] }, "seed": 110 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 83.20040692874471, "capability": 49.39850940998648, "intelligence": 15.596611891228248 }, "per_type": { "choice": { "accuracy": 0.49866666666666665, "calibration": 80.8103384555966, "cc": 35.8866269431524, "cc_by_type": { "choice": 35.8866269431524 }, "cc_tiered": 35.64074080200268, "cc_tiered_open": 36.28749068653345, "cc_tiered_sealed": 34.99399091747191, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 35.8866269431524 }, "noul": { "accuracy": 0.12, "calibration": 81.41226280393165, "cc": -76.0, "cc_by_type": { "noul": -76.0 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.34933333333333333, "calibration": 87.3786195267059, "cc": 11.246326611904923, "cc_by_type": { "score": 11.246326611904923 }, "cc_tiered": 11.149094871682063, "cc_tiered_open": 10.203316163512213, "cc_tiered_sealed": 12.094873579851914, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 11.246326611904923 } }, "raw_on_A": { "calibration": 80.48954444569493, "capability": 48.704671856760854, "intelligence": 16.919799267826768 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 2.824157802929915, "Emphasis on Accuracy 60:20:20": 2.025225537167865, "Emphasis on Cost 20:20:60": 3.299126425382793, "Emphasis on Speed 20:60:20": 3.7681689882350637, "Intelligence only": 1.4218711229563352 }, "class": "reranker", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/zeroentropy/zerank-2-reranker", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen3-Reranker-derived 4B cross-encoder", "adapter_id": "images/zerank-2/spec.json:reranker_adapter.RerankerAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.13866666666666666, "acc_among_decisive": 0.8653846153846154, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Zefan Cai (@Zefan_Cai)", "axes": { "calibration": 73.61830158128652, "cost": 33.05490478272432, "intelligence": 21.804441188912403, "speed": 77.13731150142542 }, "calibration": { "parts": { "choice": { "ece_hard": 0.1271563639749637, "ece_score": 74.56872720500726, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 74.56872720500726 }, "noul": { "abstention_rate": 0.3261455525606469, "brier": 0.22536579202900195, "ece": 0.1725448435767295, "n": 371, "score": 65.4910312846541 }, "score": { "interval90_coverage": 0.9322493224932249, "interval90_width": 0.6887985546522127, "n": 369, "nrps": 0.14871270327718153, "score": 80.79514625419819, "top_ece": 0.11769218581942746 } }, "score": 73.61830158128652 }, "capability": 47.71137138509946, "composite_ci95": { "A": [ 1.6444373649451645, 4.563766368468016 ], "B": [ 1.361709687197464, 4.0476062027996145 ], "C": [ 1.1419703923230309, 3.169282200325012 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.17041317733990147 }, "display": "Open-Jev 2B (Zefan Cai)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 27.422707291556733, "I_sealed": 16.38819706190457, "base": 21.905452176730652, "excess": 8.46112258721392, "gap": 11.034510229652163, "penalty": 0.9953887741278608, "per_type_split": { "open|choice": { "cc": 41.30120233813892, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 33.837837837837846, "hard": 26.39194715319283, "judge": 44.62755438365195, "standard": 69.8618668899121 } }, "open|noul": { "cc": 17.126293995859218, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 42.85714285714286, "hard": 30.434782608695656, "judge": -19.999999999999996, "standard": 33.33333333333333 } }, "open|score": { "cc": 23.84062554067206, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 35.74006952878567, "hard": 15.775595079225702, "judge": 40.5382915597717, "standard": 8.974465440858504 } }, "sealed|choice": { "cc": 35.268479371367654, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 41.46774074347795, "hard": 27.714932126696816, "judge": 38.54748603351957, "standard": 42.35743318142629 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 15.999999999999993, "hard": -28.155339805825243, "judge": -5.263157894736848, "standard": 12.676056338028175 } }, "sealed|score": { "cc": 13.89611181434606, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 33.49212867759059, "hard": -0.08856264182495188, "judge": 18.858024093873603, "standard": 24.62458387577451 } } } }, "jevbench_score": 3.2383990318693976, "key": "open-jev-zefan-2b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 61, "ranked": true, "ranks": { "A": 61, "B": 61, "C": 61, "capability": 54 }, "scores": { "A": 3.2383990318693976, "B": 2.798036205873629, "C": 2.248888216575971 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 1.0230181839782744, "p50_s_raw": 0.43650909198913723, "p95_s_adjusted": 1.8896679471712559, "p95_s_raw": 0.869833973585628 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 67.27712105156903, 76.2730193438662 ], "I": [ 16.64935744693625, 25.12585533438709 ], "capability": [ 42.961468081385505, 49.91119595082718 ], "composite": { "A": [ 1.6444373649451645, 4.563766368468016 ], "B": [ 1.361709687197464, 4.0476062027996145 ], "C": [ 1.1419703923230309, 3.169282200325012 ] }, "seed": 78 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 73.61830158128652, "capability": 47.71137138509946, "intelligence": 21.804441188912403 }, "per_type": { "choice": { "accuracy": 0.496, "calibration": 74.56872720500726, "cc": 35.545598363062794, "cc_by_type": { "choice": 35.545598363062794 }, "cc_tiered": 38.284840854753284, "cc_tiered_open": 41.30120233813892, "cc_tiered_sealed": 35.268479371367654, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 35.545598363062794 }, "noul": { "accuracy": 0.49866666666666665, "calibration": 65.4910312846541, "cc": -0.2666666666666706, "cc_by_type": { "noul": -0.2666666666666706 }, "cc_tiered": 8.563146997929609, "cc_tiered_open": 17.126293995859218, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.456, "calibration": 80.79514625419819, "cc": 15.96828593404923, "cc_by_type": { "score": 15.96828593404923 }, "cc_tiered": 18.86836867750906, "cc_tiered_open": 23.84062554067206, "cc_tiered_sealed": 13.89611181434606, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 15.96828593404923 } }, "raw_on_A": { "calibration": 74.44730043893902, "capability": 49.372558434366844, "intelligence": 24.297816429794665 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 2.799165550074912, "Emphasis on Accuracy 60:20:20": 2.2985053544500897, "Emphasis on Cost 20:20:60": 2.7782159796690187, "Emphasis on Speed 20:60:20": 3.6135076182350976, "Intelligence only": 1.812285794889051 }, "class": "jev-rebuild", "open": "yes", "licence": "MIT (loader); Apache-2.0 (adapter and pinned Qwen base); CC0-1.0 public training projection", "repo": "https://github.com/Zefan-Cai/Open-Jev", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "Qwen3.5-2B plus rank-8 LoRA and trained scalar decision head", "adapter_id": "images/open-jev-zefan-2b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6738544474393531, "acc_among_decisive": 0.748, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Zefan Cai", "axes": { "calibration": 83.46836347105969, "cost": 14.357426070787312, "intelligence": 52.45777412397041, "speed": 72.50871441156775 }, "calibration": { "parts": { "choice": { "ece_hard": 0.09909255777071642, "ece_score": 80.18148844585671, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 80.18148844585671 }, "noul": { "abstention_rate": 0.32884097035040427, "brier": 0.10908145798489985, "ece": 0.08491571210766313, "n": 371, "score": 83.01685757846738 }, "score": { "interval90_coverage": 0.991869918699187, "interval90_width": 0.5313911472448059, "n": 369, "nrps": 0.07069056775911403, "score": 87.20674438885496, "top_ece": 0.09258727223189336 } }, "score": 83.46836347105969 }, "capability": 67.96306879751505, "composite_ci95": { "A": [ 2.556384162487102, 2.91320674604079 ], "B": [ 2.708805295775235, 3.1474885939684447 ], "C": [ 1.7752667795049322, 2.5853832263765106 ] }, "cost": { "basis": "latest published v1.5 row (v1.5.8 combined file) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): Frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price (USD 0.42/M input, 3.00/M output) applied to 2,767,510 input tokens and zero output tokens over 1,624 decisions; self-hosted estimate, not measured hosted billing. Lineage verified by architecture; no pricing gate. The author method encodes each candidate independently.", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.7157353448275862 }, "display": "Open-Jev 27B v1.1 (Zefan Cai, LoRA + decision head on Qwen3.8-27B)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 55.382187378509315, "I_sealed": 49.533360869431505, "base": 52.45777412397041, "excess": 3.275438866639566, "gap": 5.84882650907781, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 83.22293323536725, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 75.46398238439761, "judge": 88.13447593935398, "standard": 84.93093344495604 } }, "open|noul": { "cc": 34.03933747412009, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 42.85714285714286, "hard": 65.21739130434783, "judge": -9.999999999999998, "standard": 33.33333333333333 } }, "open|score": { "cc": 48.884291426040605, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 50.782717441246206, "hard": 50.66880107494198, "judge": 50.08265733375704, "standard": 42.568510259060396 } }, "sealed|choice": { "cc": 75.06513860323663, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 74.72470623013821, "hard": 70.9841628959276, "judge": 79.51582867783985, "standard": 76.7212710924991 } }, "sealed|noul": { "cc": 22.546963950283207, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 39.99999999999999, "hard": 26.21359223300972, "judge": 18.421052631578938, "standard": 12.676056338028175 } }, "sealed|score": { "cc": 50.987980054774674, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 58.75384354090657, "hard": 44.757967203121375, "judge": 56.3751769445289, "standard": 51.48427868038395 } } } }, "jevbench_score": 2.8808671915746493, "key": "open-jev-zefan-27b-v1.1", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 62, "ranked": true, "ranks": { "A": 62, "B": 58, "C": 62, "capability": 18 }, "scores": { "A": 2.8808671915746493, "B": 3.0870580707350324, "C": 2.202117328789266 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 1.7305992921814322, "p50_s_raw": 0.7902996460907161, "p95_s_adjusted": 3.242888652905821, "p95_s_raw": 1.5464443264529106 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 79.1715005785796, 85.37530323313025 ], "I": [ 47.50165176644351, 56.56652604734154 ], "capability": [ 64.35002446863409, 69.77751267945519 ], "composite": { "A": [ 2.556384162487102, 2.91320674604079 ], "B": [ 2.708805295775235, 3.1474885939684447 ], "C": [ 1.7752667795049322, 2.5853832263765106 ] }, "seed": 77 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 83.46836347105969, "capability": 67.96306879751505, "intelligence": 52.45777412397041 }, "per_type": { "choice": { "accuracy": 0.816, "calibration": 80.18148844585671, "cc": 76.46902797381657, "cc_by_type": { "choice": 76.46902797381657 }, "cc_tiered": 79.14403591930194, "cc_tiered_open": 83.22293323536725, "cc_tiered_sealed": 75.06513860323663, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 76.46902797381657 }, "noul": { "accuracy": 0.6266666666666667, "calibration": 83.01685757846738, "cc": 25.333333333333343, "cc_by_type": { "noul": 25.333333333333343 }, "cc_tiered": 28.293150712201648, "cc_tiered_open": 34.03933747412009, "cc_tiered_sealed": 22.546963950283207, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 25.333333333333343 }, "score": { "accuracy": 0.608, "calibration": 87.20674438885496, "cc": 50.311304174784674, "cc_by_type": { "score": 50.311304174784674 }, "cc_tiered": 49.936135740407636, "cc_tiered_open": 48.884291426040605, "cc_tiered_sealed": 50.987980054774674, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 50.311304174784674 } }, "raw_on_A": { "calibration": 81.23345471191158, "capability": 67.87352962184931, "intelligence": 54.51360453178707 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 2.4131836334137757, "Emphasis on Accuracy 60:20:20": 2.9315893503528665, "Emphasis on Cost 20:20:60": 1.7049710123080262, "Emphasis on Speed 20:60:20": 3.1691813006586966, "Intelligence only": 4.3253676470159474 }, "class": "jev-rebuild", "open": "open weights", "licence": "Apache-2.0", "repo": "https://huggingface.co/ZefanCai/Open-Jev-27B-v1.1", "gpu": "RTXPRO6000", "endpoint_condition": "Self-hosted author loader, Transformers reference path on RTX PRO 6000 Blackwell; speed is a LOWER BOUND: flash-linear-attention and causal-conv1d kernels were not installed. The author documents the kernel path as answer-identical. Each candidate is encoded independently, producing high input-token usage.", "underlying": "Qwen3.8-27B (lineage verified by architecture)", "adapter_id": "/home/flori/jobs/jevbench-add-requests-20260919/r45/outputs/meta/specs/open-jev-zefan-27b-v1.1.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": "HF ZefanCai/Open-Jev-27B-v1.1@28cf73067d5b337860bbef3c85b8b82ba8730956; base Qwen/Qwen3.8-27B@1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0; author loader Zefan-Cai/Open-Jev@3308a15ccd7eea1df7a37d6ddc39b023b801ba16", "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6711590296495957, "acc_among_decisive": 0.9437751004016064, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Microsoft / neutral reproduction", "axes": { "calibration": 57.015465878675315, "cost": 53.22051180195924, "intelligence": 13.660237978082595, "speed": 91.89255362835293 }, "calibration": { "parts": { "choice": { "ece_hard": 0.24150351750278642, "ece_score": 51.69929649944272, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 51.69929649944272 }, "noul": { "abstention_rate": 0.49595687331536387, "brier": 0.28688067959784064, "ece": 0.2513310447491633, "n": 371, "score": 49.73379105016734 }, "score": { "interval90_coverage": 0.8970189701897019, "interval90_width": 0.5808491418247513, "n": 369, "nrps": 0.15574884690331758, "score": 69.6133100864159, "top_ece": 0.22599247568418224 } }, "score": 57.015465878675315 }, "capability": 35.33785192837895, "composite_ci95": { "A": [ 0.9687582607873362, 4.12745446682704 ], "B": [ 0.7192841291179243, 3.3096762413612253 ], "C": [ 0.672748792213428, 2.866287824185444 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.03625068965517241 }, "display": "Raw Phi-4 mini direct logits", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 15.681632538277219, "I_sealed": 11.638843417887971, "base": 13.660237978082595, "excess": 1.469401477951004, "gap": 4.042789120389248, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 27.48112242812705, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 26.05405405405406, "hard": 16.955017301038072, "judge": 28.80685563612393, "standard": 47.25826705734617 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -57.14285714285714, "hard": -56.52173913043479, "judge": -50.0, "standard": -66.66666666666667 } }, "open|score": { "cc": 19.563775186704607, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 51.64212383139005, "hard": 14.241949100287254, "judge": 14.5533398786592, "standard": 21.683905999264695 } }, "sealed|choice": { "cc": 25.835499598852735, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 37.47690488507872, "hard": 17.533936651583694, "judge": 26.443202979515842, "standard": 35.70636777928318 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 4.0000000000000036, "hard": -66.99029126213591, "judge": -34.210526315789465, "standard": -15.492957746478876 } }, "sealed|score": { "cc": 9.081030654811178, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 50.298756504384265, "hard": 1.511859550820871, "judge": 5.6438828552519915, "standard": 8.766231637344024 } } } }, "jevbench_score": 2.4794266829120515, "key": "raw-phi-4-mini", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 63, "ranked": true, "ranks": { "A": 63, "B": 63, "C": 63, "capability": 76 }, "scores": { "A": 2.4794266829120515, "B": 1.927494035430626, "C": 1.721824085355591 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.23357354765757918, "p50_s_raw": 0.04178677382878959, "p95_s_adjusted": 0.27689874032512307, "p95_s_raw": 0.06344937016256152 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 51.56094132127967, 61.43189419566834 ], "I": [ 9.603807805278013, 16.664196620072826 ], "capability": [ 31.514955827472072, 38.142435957298304 ], "composite": { "A": [ 0.9687582607873362, 4.12745446682704 ], "B": [ 0.7192841291179243, 3.3096762413612253 ], "C": [ 0.672748792213428, 2.866287824185444 ] }, "seed": 93 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 57.015465878675315, "capability": 35.33785192837895, "intelligence": 13.660237978082595 }, "per_type": { "choice": { "accuracy": 0.42133333333333334, "calibration": 51.69929649944272, "cc": 25.996798120553574, "cc_by_type": { "choice": 25.996798120553574 }, "cc_tiered": 26.658311013489893, "cc_tiered_open": 27.48112242812705, "cc_tiered_sealed": 25.835499598852735, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 25.996798120553574 }, "noul": { "accuracy": 0.304, "calibration": 49.73379105016734, "cc": -39.2, "cc_by_type": { "noul": -39.2 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.4026666666666667, "calibration": 69.6133100864159, "cc": 13.214800471337696, "cc_by_type": { "score": 13.214800471337696 }, "cc_tiered": 14.322402920757892, "cc_tiered_open": 19.563775186704607, "cc_tiered_sealed": 9.081030654811178, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 13.214800471337696 } }, "raw_on_A": { "calibration": 53.16092441580774, "capability": 33.04410281531054, "intelligence": 12.927281214813327 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 2.1766003082696974, "Emphasis on Accuracy 60:20:20": 1.497083363151444, "Emphasis on Cost 20:20:60": 2.657076827352907, "Emphasis on Speed 20:60:20": 2.994215323690922, "Intelligence only": 1.0196116461240732 }, "class": "raw-logit-control", "open": "yes", "licence": "MIT", "repo": "https://huggingface.co/microsoft/Phi-4-mini-instruct", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Phi-4-mini-instruct BF16", "adapter_id": "images/raw-phi-4-mini/spec.json:raw_adapter.RawControlAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5040431266846361, "acc_among_decisive": 0.6096256684491979, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "sabeel111 / Featherless AI", "axes": { "calibration": 50.197881904888476, "cost": 70.31175519898045, "intelligence": 13.000426205687345, "speed": 58.929227255635794 }, "calibration": { "parts": { "choice": { "ece_hard": 0.22436826460136267, "ece_score": 55.126347079727466, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 55.126347079727466 }, "noul": { "abstention_rate": 0.0, "brier": 0.40897819205718194, "ece": 0.4071351151116132, "n": 371, "score": 18.572976977677357 }, "score": { "interval90_coverage": 0.8807588075880759, "interval90_width": 0.6312104787714541, "n": 369, "nrps": 0.17353025104683517, "score": 76.89432165726062, "top_ece": 0.14429165790397622 } }, "score": 50.197881904888476 }, "capability": 31.59915405528791, "composite_ci95": { "A": [ 0.709004207843539, 4.446455346949053 ], "B": [ 0.5274465681104142, 3.6235557003922736 ], "C": [ 0.49236403322467986, 3.0878162131590647 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.009763559113300492 }, "display": "SimpleJev (Qwen3.5-0.8B, CPU)", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 17.1749859973889, "I_sealed": 8.82586641398579, "base": 13.000426205687345, "excess": 5.775731940964869, "gap": 8.349119583403112, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 16.101545239125173, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 2.702702702702708, "hard": 13.180245360176167, "judge": 28.80685563612393, "standard": 9.585600669736289 } }, "open|noul": { "cc": 31.332643202208423, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": 30.434782608695656, "judge": 30.000000000000004, "standard": 22.222222222222232 } }, "open|score": { "cc": 4.0907695508331114, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 21.659111114444617, "hard": -13.376366573006472, "judge": 24.5874654817379, "standard": -0.504172879650655 } }, "sealed|choice": { "cc": 16.39171901424465, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 10.87133249575052, "hard": 9.898190045248851, "judge": 19.92551210428307, "standard": 26.838280576425678 } }, "sealed|noul": { "cc": 10.085880227712721, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 24.0, "hard": 14.563106796116498, "judge": 5.263157894736836, "standard": 1.4084507042253502 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 19.829285077433635, "hard": -8.603274422314232, "judge": 2.232240541966324, "standard": -0.6083842924782035 } } } }, "jevbench_score": 2.112084174105171, "key": "simplejev-qwen3.5-0.8b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 64, "ranked": true, "ranks": { "A": 64, "B": 64, "C": 64, "capability": 81 }, "scores": { "A": 2.112084174105171, "B": 1.649257467523907, "C": 1.4667251209063683 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 8.797023160476238, "p50_s_raw": 4.323511580238119, "p95_s_adjusted": 14.545931534096598, "p95_s_raw": 7.197965767048299 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 44.436316357034364, 54.56938440080917 ], "I": [ 8.668681285963862, 17.344962779702836 ], "capability": [ 27.059726936224934, 35.62162347397245 ], "composite": { "A": [ 0.709004207843539, 4.446455346949053 ], "B": [ 0.5274465681104142, 3.6235557003922736 ], "C": [ 0.49236403322467986, 3.0878162131590647 ] }, "seed": 100 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 50.197881904888476, "capability": 31.59915405528791, "intelligence": 13.000426205687345 }, "per_type": { "choice": { "accuracy": 0.3373333333333333, "calibration": 55.126347079727466, "cc": 15.254397847730704, "cc_by_type": { "choice": 15.254397847730704 }, "cc_tiered": 16.24663212668491, "cc_tiered_open": 16.101545239125173, "cc_tiered_sealed": 16.39171901424465, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 15.254397847730704 }, "noul": { "accuracy": 0.576, "calibration": 18.572976977677357, "cc": 15.199999999999992, "cc_by_type": { "noul": 15.199999999999992 }, "cc_tiered": 20.709261714960572, "cc_tiered_open": 31.332643202208423, "cc_tiered_sealed": 10.085880227712721, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 15.199999999999992 }, "score": { "accuracy": 0.33066666666666666, "calibration": 76.89432165726062, "cc": 0.21568120093692356, "cc_by_type": { "score": 0.21568120093692356 }, "cc_tiered": 2.0453847754165557, "cc_tiered_open": 4.0907695508331114, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.21568120093692356 } }, "raw_on_A": { "calibration": 52.11123764887125, "capability": 32.38563147006098, "intelligence": 12.66002529125071 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 1.8759479039856477, "Emphasis on Accuracy 60:20:20": 1.2903897195924547, "Emphasis on Cost 20:20:60": 2.475315897383538, "Emphasis on Speed 20:60:20": 2.3795755523953557, "Intelligence only": 0.8788864373471849 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 server; Apache-2.0 Qwen3.5-0.8B checkpoint", "repo": "https://github.com/featherless-ai/simple-jev", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "underlying": "Qwen3.5-0.8B through SimpleJev native assistant-prefill option-logit scorer", "adapter_id": "images/simplejev-qwen3.5-0.8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 1.0, "acc_among_decisive": 0.5822102425876011, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Aditya (isHeSatoshi)", "axes": { "calibration": 77.23092648324078, "cost": 60.77469204714053, "intelligence": 8.670533657732655, "speed": 90.21131723708834 }, "calibration": { "parts": { "choice": { "ece_hard": 0.1267499372828752, "ece_score": 74.65001254342496, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 74.65001254342496 }, "noul": { "abstention_rate": 0.7653333333333333, "brier": 0.252171281098217, "ece": 0.15089599377413593, "n": 375, "score": 69.82080124517282 }, "score": { "interval90_coverage": 0.9733333333333334, "interval90_width": 0.8366666666666663, "n": 375, "nrps": 0.15538624289679162, "score": 87.22196566112457, "top_ece": 0.05008722194035847 } }, "score": 77.23092648324078 }, "capability": 42.95073007048671, "composite_ci95": { "A": [ 0.2465766163795243, 1.6433530428400023 ], "B": [ 0.17030179892922623, 1.2180129321235638 ], "C": [ 0.17123376137466964, 1.1412173908611125 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.020300665024630543 }, "display": "smalljev semantic-v9", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 10.044883698522286, "I_sealed": 7.296183616943025, "base": 8.670533657732655, "excess": 0.17531243914101724, "gap": 2.748700081579261, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 18.8258695634495, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 29.945945945945947, "hard": 13.180245360176167, "judge": 28.80685563612393, "standard": 9.585600669736289 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -71.42857142857143, "hard": -73.91304347826086, "judge": -80.0, "standard": -66.66666666666667 } }, "open|score": { "cc": 11.308781532117358, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 1.465713661728496, "hard": 15.552501797814056, "judge": 21.821381656800277, "standard": -8.026025251105985 } }, "sealed|choice": { "cc": 12.092078018745205, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 37.47690488507872, "hard": 8.37104072398188, "judge": 6.890130353817524, "standard": 14.644660672496622 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -60.0, "hard": -82.52427184466019, "judge": -52.63157894736843, "standard": -52.112676056338024 } }, "sealed|score": { "cc": 9.796472832083872, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 12.348066695721426, "hard": 6.731683495180563, "judge": 16.491346453277444, "standard": 4.607944142281362 } } } }, "jevbench_score": 0.7719455446134506, "key": "smalljev", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 65, "ranked": true, "ranks": { "A": 65, "B": 65, "C": 65, "capability": 65 }, "scores": { "A": 0.7719455446134506, "B": 0.5545056442894855, "C": 0.5360732948704519 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2831438841298223, "p50_s_raw": 0.06657194206491113, "p95_s_adjusted": 0.33640395337715745, "p95_s_raw": 0.09320197668857871 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 71.4661928849833, 79.90983904292283 ], "I": [ 5.751868300889253, 11.443348446757808 ], "capability": [ 39.42005219873977, 44.84742177133798 ], "composite": { "A": [ 0.2465766163795243, 1.6433530428400023 ], "B": [ 0.17030179892922623, 1.2180129321235638 ], "C": [ 0.17123376137466964, 1.1412173908611125 ] }, "seed": 101 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 77.23092648324078, "capability": 42.95073007048671, "intelligence": 8.670533657732655 }, "per_type": { "choice": { "accuracy": 0.336, "calibration": 74.65001254342496, "cc": 15.0838835576859, "cc_by_type": { "choice": 15.0838835576859 }, "cc_tiered": 15.458973791097353, "cc_tiered_open": 18.8258695634495, "cc_tiered_sealed": 12.092078018745205, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 15.0838835576859 }, "noul": { "accuracy": 0.17066666666666666, "calibration": 69.82080124517282, "cc": -65.86666666666667, "cc_by_type": { "noul": -65.86666666666667 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.3626666666666667, "calibration": 87.22196566112457, "cc": 9.603703249850426, "cc_by_type": { "score": 9.603703249850426 }, "cc_tiered": 10.552627182100615, "cc_tiered_open": 11.308781532117358, "cc_tiered_sealed": 9.796472832083872, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 9.603703249850426 } }, "raw_on_A": { "calibration": 71.91610164610462, "capability": 40.0721638349662, "intelligence": 8.22822602382777 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.6314288092197278, "Emphasis on Accuracy 60:20:20": 0.4025186185800094, "Emphasis on Cost 20:20:60": 0.8553620766195851, "Emphasis on Speed 20:60:20": 0.9110150146047675, "Intelligence only": 0.2607338855207101 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0", "repo": "https://github.com/isHeSatoshi/smalljev", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "MiniCPM5-2B-Base, 2.5B dense, with LoRA and native semantic decision heads", "adapter_id": "images/smalljev/spec.json:remote_inproc_v15.RemoteInprocV15Adapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.23466666666666666, "acc_among_decisive": 0.7272727272727273, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Jared Palmer", "axes": { "calibration": 66.26655558235618, "cost": 80.09400261254245, "intelligence": 7.598789169415934, "speed": 91.51497362122684 }, "calibration": { "parts": { "choice": { "ece_hard": 0.20778793892461145, "ece_score": 58.44241221507771, "mean_tvd": null, "n": 731, "n_ece_hard": 306, "n_tvd": 0, "score": 58.44241221507771 }, "noul": { "abstention_rate": 0.568733153638814, "brier": 0.2628086253369272, "ece": 0.18765498652291104, "n": 371, "score": 62.4690026954178 }, "score": { "interval90_coverage": 0.9320652173913043, "interval90_width": 0.8045742753623186, "n": 368, "nrps": 0.17410404691603434, "score": 77.88825183657305, "top_ece": 0.1340654581762524 } }, "score": 66.26655558235618 }, "capability": 36.93267237588606, "composite_ci95": { "A": [ 0.13850323393885078, 1.4591062434238613 ], "B": [ 0.09387316749385785, 1.070943045752662 ], "C": [ 0.09618280134642418, 1.0132682245999038 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.004608220443349753 }, "display": "kev 0.6B (research preview)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 8.945068603011505, "I_sealed": 6.252509735820362, "base": 7.598789169415934, "excess": 0.1191712247528991, "gap": 2.692558867191143, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 16.179258482292497, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 45.513513513513516, "hard": 9.405473419314262, "judge": 24.851680949241928, "standard": 2.051067392214312 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 0.0, "hard": -73.91304347826086, "judge": -60.0, "standard": -44.44444444444444 } }, "open|score": { "cc": 10.655947326742021, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 23.83763830928548, "hard": -1.1034131190897023, "judge": 28.04844552251834, "standard": 1.4950754334692617 } }, "sealed|choice": { "cc": 18.757529207461086, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 34.8163476461459, "hard": 6.334841628959255, "judge": 24.581005586592198, "standard": 26.838280576425678 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -4.0000000000000036, "hard": -66.99029126213591, "judge": -44.73684210526315, "standard": -15.492957746478876 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 11.09665079983172, "hard": -14.21346816054152, "judge": 12.732793638645301, "standard": -1.8098672811444283 } } } }, "jevbench_score": 0.5431213026023877, "key": "kev-0.6b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 66, "ranked": true, "ranks": { "A": 66, "B": 66, "C": 66, "capability": 74 }, "scores": { "A": 0.5431213026023877, "B": 0.382771242886796, "C": 0.377167571251658 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.26348884413018825, "p50_s_raw": 0.05674442206509411, "p95_s_adjusted": 0.2677567361854017, "p95_s_raw": 0.05887836809270084 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 60.54900964119034, 70.35051467735538 ], "I": [ 4.680395319947538, 10.897085922837281 ], "capability": [ 33.22958246000976, 39.72314225130261 ], "composite": { "A": [ 0.13850323393885078, 1.4591062434238613 ], "B": [ 0.09387316749385785, 1.070943045752662 ], "C": [ 0.09618280134642418, 1.0132682245999038 ] }, "seed": 53 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 66.26655558235618, "capability": 36.93267237588606, "intelligence": 7.598789169415934 }, "per_type": { "choice": { "accuracy": 0.368, "calibration": 58.44241221507771, "cc": 19.176226518761275, "cc_by_type": { "choice": 19.176226518761275 }, "cc_tiered": 17.468393844876793, "cc_tiered_open": 16.179258482292497, "cc_tiered_sealed": 18.757529207461086, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 19.176226518761275 }, "noul": { "accuracy": 0.296, "calibration": 62.4690026954178, "cc": -40.800000000000004, "cc_by_type": { "noul": -40.800000000000004 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.30933333333333335, "calibration": 77.88825183657305, "cc": -0.08941736278893497, "cc_by_type": { "score": -0.08941736278893497 }, "cc_tiered": 5.327973663371011, "cc_tiered_open": 10.655947326742021, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 65.17717796738081, "capability": 37.13879390532707, "intelligence": 9.100409843273326 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.02533333333333332, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.01866666666666672, "n": 375 } }, "views": { "Balanced 33:33:33": 0.4469959095569562, "Emphasis on Accuracy 60:20:20": 0.2761353719898703, "Emphasis on Cost 20:20:60": 0.6416336282227073, "Emphasis on Speed 20:60:20": 0.6529389194958221, "Intelligence only": 0.17550648828077675 }, "class": "jev-rebuild", "open": true, "licence": "Apache-2.0", "repo": "https://github.com/jaredpalmer/kev", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen3-0.6B-Base + LoRA + learned pointer head; jaredpalmer/kev-0.6b", "adapter_id": "images/kev-0.6b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.431266846361186, "acc_among_decisive": 0.69375, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Deepan Wadhwa", "axes": { "calibration": 68.56606448002526, "cost": 79.10704862374335, "intelligence": 7.425225288997898, "speed": 87.12856375997052 }, "calibration": { "parts": { "choice": { "ece_hard": 0.18936051642522217, "ece_score": 62.12789671495557, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 62.12789671495557 }, "noul": { "abstention_rate": 0.4933333333333333, "brier": 0.2702030712841029, "ece": 0.17767917616168663, "n": 375, "score": 64.46416476766268 }, "score": { "interval90_coverage": 0.9706666666666667, "interval90_width": 0.8675555555555556, "n": 375, "nrps": 0.1718997562232394, "score": 79.10613195745756, "top_ece": 0.12298880231380463 } }, "score": 68.56606448002526 }, "capability": 37.99564488451158, "composite_ci95": { "A": [ 0.20924922812147978, 1.114873836219044 ], "B": [ 0.14364373295734356, 0.808381046176735 ], "C": [ 0.14531196397324977, 0.7742179418187807 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.004970862068965518 }, "display": "OpenDecision (ModernBERT-large zero-shot)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 8.199956224645156, "I_sealed": 6.6504943533506395, "base": 7.425225288997898, "excess": -1.0239257711437273, "gap": 1.5494618712945165, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 21.530744131863024, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 37.72972972972973, "hard": -3.8062283737024116, "judge": 32.76203032300594, "standard": 47.25826705734617 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -14.28571428571429, "hard": -82.6086956521739, "judge": -30.000000000000004, "standard": -22.22222222222222 } }, "open|score": { "cc": 3.069124542072446, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 29.089033648823246, "hard": -4.629529940171362, "judge": 10.925488522812966, "standard": -6.32806701792612 } }, "sealed|choice": { "cc": 18.786220529284186, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 34.8163476461459, "hard": 9.898190045248851, "judge": 19.92551210428307, "standard": 26.838280576425678 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 12.00000000000001, "hard": -39.80582524271845, "judge": -50.0, "standard": -15.492957746478876 } }, "sealed|score": { "cc": 1.165262530767735, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 23.523223159689433, "hard": -5.374879041668379, "judge": 2.0574839819155533, "standard": 1.728233184457384 } } } }, "jevbench_score": 0.508795165212109, "key": "opendecision", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 67, "ranked": true, "ranks": { "A": 67, "B": 67, "C": 67, "capability": 71 }, "scores": { "A": 0.508795165212109, "B": 0.3579489209247008, "C": 0.3533299758417423 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.29369962783530357, "p50_s_raw": 0.07184981391765177, "p95_s_adjusted": 0.6595386148197576, "p95_s_raw": 0.2547693074098788 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 62.90521170003539, 71.97774479185684 ], "I": [ 5.421041180341186, 9.877186440823028 ], "capability": [ 34.738025189088674, 40.24525966167053 ], "composite": { "A": [ 0.20924922812147978, 1.114873836219044 ], "B": [ 0.14364373295734356, 0.808381046176735 ], "C": [ 0.14531196397324977, 0.7742179418187807 ] }, "seed": 80 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 68.56606448002526, "capability": 37.99564488451158, "intelligence": 7.425225288997898 }, "per_type": { "choice": { "accuracy": 0.36933333333333335, "calibration": 62.12789671495557, "cc": 19.346740808806086, "cc_by_type": { "choice": 19.346740808806086 }, "cc_tiered": 20.158482330573605, "cc_tiered_open": 21.530744131863024, "cc_tiered_sealed": 18.786220529284186, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 19.346740808806086 }, "noul": { "accuracy": 0.3466666666666667, "calibration": 64.46416476766268, "cc": -30.666666666666664, "cc_by_type": { "noul": -30.666666666666664 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.3253333333333333, "calibration": 79.10613195745756, "cc": 2.4048951483054193, "cc_by_type": { "score": 2.4048951483054193 }, "cc_tiered": 2.1171935364200904, "cc_tiered_open": 3.069124542072446, "cc_tiered_sealed": 1.165262530767735, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 2.4048951483054193 } }, "raw_on_A": { "calibration": 66.1740117306899, "capability": 37.492707526259565, "intelligence": 8.811403321829234 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.41664409033845246, "Emphasis on Accuracy 60:20:20": 0.25754719918644864, "Emphasis on Cost 20:20:60": 0.5990327904113044, "Emphasis on Speed 20:60:20": 0.6067044473606604, "Intelligence only": 0.1637528610901445 }, "class": "classifier", "open": "yes", "licence": "Apache-2.0", "repo": "https://github.com/deepanwadhwa/OpenDecision", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "MoritzLaurer/ModernBERT-large-zeroshot-v2.0 (~400M) used as a zero-shot NLI decision engine", "adapter_id": "images/opendecision/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5066666666666667, "acc_among_decisive": 0.6842105263157895, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Mourad Ghafiri", "axes": { "calibration": 77.95657740696296, "cost": 79.5180630553356, "intelligence": 7.297270616183145, "speed": 55.90747410573378 }, "calibration": { "parts": { "choice": { "ece_hard": 0.16652875000000003, "ece_score": 66.69425, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 66.69425 }, "noul": { "abstention_rate": 0.7813333333333333, "brier": 0.20344673173333316, "ece": 0.05893599999999998, "n": 375, "score": 88.2128 }, "score": { "interval90_coverage": 0.8586666666666667, "interval90_width": 0.6119999999999997, "n": 375, "nrps": 0.16765142224888893, "score": 78.9626822208889, "top_ece": 0.12654746666666666 } }, "score": 77.95657740696296 }, "capability": 42.62692401157305, "composite_ci95": { "A": [ 0.18025356076996352, 1.2178442350406786 ], "B": [ 0.12404410707015784, 0.9011760636632091 ], "C": [ 0.1251760838680302, 0.8457251632226935 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.00481649630541872 }, "display": "Qwen3.5-0.8B Decision Model (Mourad Ghafiri)", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 4.79058394567308, "I_sealed": 9.803957286693208, "base": 7.297270616183145, "excess": -7.5867609834583725, "gap": -5.013373341020128, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 13.339251237806783, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": -1.1891891891891837, "hard": 13.180245360176167, "judge": 20.896506262359924, "standard": 9.585600669736289 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -28.57142857142857, "hard": -73.91304347826086, "judge": -90.0, "standard": -88.88888888888889 } }, "open|score": { "cc": 1.0325005992124563, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 40.642727272727264, "hard": -20.778611111111122, "judge": 25.48779661016949, "standard": -11.83333333333334 } }, "sealed|choice": { "cc": 21.186916712409108, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 24.174118690414627, "hard": 20.079185520361975, "judge": 20.856610800744896, "standard": 22.404236974996934 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -60.0, "hard": -68.93203883495146, "judge": -57.89473684210527, "standard": -43.66197183098591 } }, "sealed|score": { "cc": 8.224955147670517, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 42.56900709219856, "hard": 1.4315441176470767, "judge": 8.896227544910184, "standard": 3.6328426395938807 } } } }, "jevbench_score": 0.47247471382875306, "key": "mghafiri-qwen3.5-0.8b-decision-model", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 68, "ranked": true, "ranks": { "A": 68, "B": 68, "C": 68, "capability": 66 }, "scores": { "A": 0.47247471382875306, "B": 0.3355765591935497, "C": 0.3281074401588563 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 11.252091223932803, "p50_s_raw": 5.551045611966401, "p95_s_adjusted": 22.804436485376208, "p95_s_raw": 11.327218242688105 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 71.68948470123519, 80.02198153079908 ], "I": [ 5.167522017117285, 10.33633269787795 ], "capability": [ 38.95895177314971, 44.525366548573615 ], "composite": { "A": [ 0.18025356076996352, 1.2178442350406786 ], "B": [ 0.12404410707015784, 0.9011760636632091 ], "C": [ 0.1251760838680302, 0.8457251632226935 ] }, "seed": 67 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 77.95657740696296, "capability": 42.62692401157305, "intelligence": 7.297270616183145 }, "per_type": { "choice": { "accuracy": 0.36933333333333335, "calibration": 66.69425, "cc": 19.346740808806086, "cc_by_type": { "choice": 19.346740808806086 }, "cc_tiered": 17.263083975107946, "cc_tiered_open": 13.339251237806783, "cc_tiered_sealed": 21.186916712409108, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 19.346740808806086 }, "noul": { "accuracy": 0.192, "calibration": 88.2128, "cc": -61.6, "cc_by_type": { "noul": -61.6 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.4106666666666667, "calibration": 78.9626822208889, "cc": 7.9552982456141175, "cc_by_type": { "score": 7.9552982456141175 }, "cc_tiered": 4.628727873441487, "cc_tiered_open": 1.0325005992124563, "cc_tiered_sealed": 8.224955147670517, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 7.9552982456141175 } }, "raw_on_A": { "calibration": 67.24894909368233, "capability": 38.149863284895126, "intelligence": 9.050777476107926 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.381493687842595, "Emphasis on Accuracy 60:20:20": 0.24118281186608428, "Emphasis on Cost 20:20:60": 0.5528132957063066, "Emphasis on Speed 20:60:20": 0.5239265172020665, "Intelligence only": 0.15543232661348225 }, "class": "jev-rebuild", "open": "yes", "licence": "MIT code and training data; Apache-2.0 model weights", "repo": "https://huggingface.co/mghafiri/qwen3.5-0.8B-decision-model", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "underlying": "Qwen3.5-0.8B-Base fine-tuned as a JevLite decision model with per-question calibration", "adapter_id": "images/mghafiri-qwen3.5-0.8b-decision-model/spec.json:mghafiri_adapter.JevLite", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.21866666666666668, "acc_among_decisive": 0.8780487804878049, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "LibertAI", "axes": { "calibration": 29.6278299611924, "cost": 80.28784730320886, "intelligence": 6.531204174677637, "speed": 85.44505463939521 }, "calibration": { "parts": { "choice": { "ece_hard": 0.5370261325853646, "ece_score": 0.0, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 0.0 }, "noul": { "abstention_rate": 0.2264150943396226, "brier": 0.34735578114832993, "ece": 0.30478924226600856, "n": 371, "score": 39.04215154679829 }, "score": { "interval90_coverage": 0.6233062330623306, "interval90_width": 0.4112466124661248, "n": 369, "nrps": 0.24237776576192704, "score": 49.84133833677892, "top_ece": 0.3803977337512473 } }, "score": 29.6278299611924 }, "capability": 18.07951706793502, "composite_ci95": { "A": [ 0.12746211676468558, 0.7725020253140645 ], "B": [ 0.08887678609852033, 0.587439805908629 ], "C": [ 0.08851535886436501, 0.5364597398014336 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: documented hosted-model estimate. Frozen market reference deepinfra:Qwen/Qwen3.5-0.8B.", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.004540166256157636 }, "display": "Deem 0.8B v1", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 8.352332961291632, "I_sealed": 4.710075388063642, "base": 6.531204174677637, "excess": 1.0688699307897456, "gap": 3.6422575732279894, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 25.056998883874897, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 41.621621621621614, "hard": 9.405473419314262, "judge": 40.672379696769944, "standard": 24.654667224780237 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 0.0, "hard": -39.13043478260869, "judge": -9.999999999999998, "standard": 11.111111111111116 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 23.118548982848488, "hard": -12.673038757528188, "judge": 7.843499385378882, "standard": -67.26014388289686 } }, "sealed|choice": { "cc": 14.130226164190928, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 32.15579040721308, "hard": 3.7895927601809762, "judge": 17.132216014897597, "standard": 21.29572607463975 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 28.000000000000004, "hard": -39.80582524271845, "judge": -2.631578947368418, "standard": 4.225352112676051 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -7.688289022450467, "hard": -26.74875998927384, "judge": -33.081610869032005, "standard": -11.930482263268317 } } } }, "jevbench_score": 0.32342918922131697, "key": "deem-0.8-v1", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 69, "ranked": true, "ranks": { "A": 69, "B": 69, "C": 69, "capability": 89 }, "scores": { "A": 0.32342918922131697, "B": 0.23429153017801835, "C": 0.2246036036259146 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.4647796718403697, "p50_s_raw": 0.15738983592018485, "p95_s_adjusted": 0.6141117558348923, "p95_s_raw": 0.23205587791744614 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 25.404113930608833, 32.82152927572935 ], "I": [ 4.660400387407388, 9.084320195556709 ], "capability": [ 15.752622734868103, 20.176991128075684 ], "composite": { "A": [ 0.12746211676468558, 0.7725020253140645 ], "B": [ 0.08887678609852033, 0.587439805908629 ], "C": [ 0.08851535886436501, 0.5364597398014336 ] }, "seed": 35 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 29.6278299611924, "capability": 18.07951706793502, "intelligence": 6.531204174677637 }, "per_type": { "choice": { "accuracy": 0.34933333333333333, "calibration": 0.0, "cc": 16.78902645813397, "cc_by_type": { "choice": 16.78902645813397 }, "cc_tiered": 19.593612524032913, "cc_tiered_open": 25.056998883874897, "cc_tiered_sealed": 14.130226164190928, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 16.78902645813397 }, "noul": { "accuracy": 0.4533333333333333, "calibration": 39.04215154679829, "cc": -9.333333333333337, "cc_by_type": { "noul": -9.333333333333337 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.30933333333333335, "calibration": 49.84133833677892, "cc": -19.150447666018742, "cc_by_type": { "score": -19.150447666018742 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 29.6549349776408, "capability": 17.995438687555687, "intelligence": 6.335942397470573 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 0.28875743313428337, "Emphasis on Accuracy 60:20:20": 0.17645226737338762, "Emphasis on Cost 20:20:60": 0.4219666221718261, "Emphasis on Speed 20:60:20": 0.4251280933682276, "Intelligence only": 0.11143965867359962 }, "class": "system-one-open", "open": "yes", "licence": "Not independently recorded", "repo": "https://huggingface.co/LibertAIDAI/deem-0.8-v1", "gpu": "H100", "endpoint_condition": "evaluator-owned H100 GPU", "underlying": "LibertAIDAI/deem-0.8-v1 @ 8cbabbb2c4a7ef13c6b43f0ef3ae4157983c6d21; root model.safetensors SHA-256 80438125177c0681e855158d46720cbbd4ff07ea620cd55b0f456aafbfcf778d. Author Python GPU server, default DEEM_N_ORDERS=1, raw softmax (temperature 1.0). A four-field shim translates the wire format. The submitted interface accepts choice labels without descriptions and noul instructions without criteria; score receives full level descriptions. The author Rust/AVX-512 latency claim was not measured.", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7735849056603774, "acc_among_decisive": 0.5923344947735192, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "FlyMy.AI (@denti)", "axes": { "calibration": 75.07737238688318, "cost": 80.08651867745036, "intelligence": 6.110691078442272, "speed": 91.472185793536 }, "calibration": { "parts": { "choice": { "ece_hard": 0.1702515239808779, "ece_score": 65.94969520382442, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 65.94969520382442 }, "noul": { "abstention_rate": 0.8625336927223719, "brier": 0.2408499885802099, "ece": 0.11364094419260821, "n": 371, "score": 77.27181116147835 }, "score": { "interval90_coverage": 0.959349593495935, "interval90_width": 0.8803071364046972, "n": 369, "nrps": 0.17723101114191298, "score": 82.01061079534679, "top_ece": 0.09127838647557562 } }, "score": 75.07737238688318 }, "capability": 40.59403173266273, "composite_ci95": { "A": [ 0.11705708633042304, 0.8235356910346963 ], "B": [ 0.07857381222727884, 0.5860129975515587 ], "C": [ 0.08128964328501598, 0.5718997854407614 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.004610868226600985 }, "display": "Decision Fast (FlyMy.AI, v53a)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 5.162912545807329, "I_sealed": 7.058469611077215, "base": 6.110691078442272, "excess": -4.468944707708131, "gap": -1.8955570652698865, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 15.371451872027388, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 29.945945945945947, "hard": 3.743315508021402, "judge": 24.851680949241928, "standard": 17.120133947258267 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -14.28571428571429, "hard": -100.0, "judge": -90.0, "standard": -88.88888888888889 } }, "open|score": { "cc": 0.1172857653946, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 37.515372511743564, "hard": -11.188275382721603, "judge": -4.115171438893084, "standard": 10.37805049488405 } }, "sealed|choice": { "cc": 21.175408833231646, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 32.15579040721308, "hard": 4.298642533936631, "judge": 27.374301675977662, "standard": 40.14041138071193 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -16.000000000000004, "hard": -88.3495145631068, "judge": -86.8421052631579, "standard": -83.09859154929578 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 22.391298609400668, "hard": -12.696561777971827, "judge": 3.397122091897453, "standard": -2.490577451116116 } } } }, "jevbench_score": 0.2981489585308357, "key": "decision-fast", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 70, "ranked": true, "ranks": { "A": 70, "B": 70, "C": 70, "capability": 68 }, "scores": { "A": 0.2981489585308357, "B": 0.20514889729513758, "C": 0.2070478878686359 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2653681320138276, "p50_s_raw": 0.05768406600691378, "p95_s_adjusted": 0.26849280574824663, "p95_s_raw": 0.059246402874123305 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 69.51262824417705, 78.1703871756354 ], "I": [ 4.394274632210615, 8.794694293942209 ], "capability": [ 37.53859057048145, 42.929975748521436 ], "composite": { "A": [ 0.11705708633042304, 0.8235356910346963 ], "B": [ 0.07857381222727884, 0.5860129975515587 ], "C": [ 0.08128964328501598, 0.5718997854407614 ] }, "seed": 32 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 75.07737238688318, "capability": 40.59403173266273, "intelligence": 6.110691078442272 }, "per_type": { "choice": { "accuracy": 0.37333333333333335, "calibration": 65.94969520382442, "cc": 19.85828367894051, "cc_by_type": { "choice": 19.85828367894051 }, "cc_tiered": 18.273430352629518, "cc_tiered_open": 15.371451872027388, "cc_tiered_sealed": 21.175408833231646, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 19.85828367894051 }, "noul": { "accuracy": 0.12266666666666666, "calibration": 77.27181116147835, "cc": -75.46666666666667, "cc_by_type": { "noul": -75.46666666666667 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.31733333333333336, "calibration": 82.01061079534679, "cc": -1.6484760263063114, "cc_by_type": { "score": -1.6484760263063114 }, "cc_tiered": 0.0586428826973, "cc_tiered_open": 0.1172857653946, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 74.21190941158237, "capability": 40.84165753605771, "intelligence": 7.471405660533052 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 0.23953342944478384, "Emphasis on Accuracy 60:20:20": 0.14519180403822335, "Emphasis on Cost 20:20:60": 0.3522038654235207, "Emphasis on Speed 20:60:20": 0.35744386580296805, "Intelligence only": 0.09127061519323529 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 notices on the included code and the pinned base; the weights are an evaluation preview under EVALUATION-PERMISSION.md, not a cleared commercial release", "repo": "https://huggingface.co/flymy-ai/decision-fast-preview", "gpu": "RTX6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container", "underlying": "Qwen/Qwen3-0.6B-Base with a trained LoRA adapter and pointer head (10.6M trainable parameters)", "adapter_id": "images/decision-fast/spec.json:flymy_v15.FlyMyPackage", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-05", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.13746630727762804, "acc_among_decisive": 0.9019607843137255, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Chinh Nguyen (ncchinh)", "axes": { "calibration": 73.02134054125926, "cost": 89.11296503596958, "intelligence": 6.054813340422714, "speed": 94.90893356488051 }, "calibration": { "parts": { "choice": { "ece_hard": 0.1464534374999999, "ece_score": 70.70931250000002, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 70.70931250000002 }, "noul": { "abstention_rate": 0.72, "brier": 0.2701432261600001, "ece": 0.15907813333333334, "n": 375, "score": 68.18437333333334 }, "score": { "interval90_coverage": 0.9653333333333334, "interval90_width": 0.8146666666666667, "n": 375, "nrps": 0.1639986175244445, "score": 80.17033579044444, "top_ece": 0.11629733333333336 } }, "score": 73.02134054125926 }, "capability": 39.53807694084099, "composite_ci95": { "A": [ 0.0550759146094291, 0.8570777384335173 ], "B": [ 0.03640384449772412, 0.6085773029152636 ], "C": [ 0.03824716292321466, 0.5951928739121649 ] }, "cost": { "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): DeepInfra base-size encoders (bge-base, e5-base, gte-base, all-mpnet-base) $0.005/M input (ModernBERT-base has no market listing; base-size encoder reference) x the package's own measured usage.input_tokens; one forward pass, nothing generated; run in process on 1x H100 80GB (Lium), quyet 1.0.0; estimated, not charged", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0023062222906403943 }, "display": "Quyet-1.0-Small-EN (Chinh Nguyen, ModernBERT-base encoder with fixed heads)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 5.966953908358329, "I_sealed": 6.142672772487099, "base": 6.054813340422714, "excess": -2.749106506567014, "gap": -0.17571886412877014, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 11.596876448581234, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 10.486486486486491, "hard": 24.50456118276188, "judge": 1.1206328279499007, "standard": 2.051067392214312 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -71.42857142857143, "hard": -65.21739130434783, "judge": -70.0, "standard": -44.44444444444444 } }, "open|score": { "cc": 6.303985276493753, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 12.853939393939395, "hard": 15.763981481481492, "judge": 4.772711864406753, "standard": -13.594074074074047 } }, "sealed|choice": { "cc": 14.421677328746334, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 4.219939398418472, "hard": 11.934389140271476, "judge": 14.338919925512121, "standard": 24.621258775711308 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -48.0, "hard": -84.46601941747574, "judge": -52.63157894736843, "standard": -60.56338028169015 } }, "sealed|score": { "cc": 4.006340988714965, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 17.547730496453894, "hard": 1.5428921568626852, "judge": 5.2876646706586765, "standard": 0.240558375634492 } } } }, "jevbench_score": 0.29239342204088814, "key": "quyet-1-0-small-en", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 71, "ranked": true, "ranks": { "A": 71, "B": 72, "C": 71, "capability": 70 }, "scores": { "A": 0.29239342204088814, "B": 0.20045885649425463, "C": 0.20305098752839457 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.1791461439919658, "p50_s_raw": 0.0145730719959829, "p95_s_adjusted": 0.18025992200709878, "p95_s_raw": 0.015129961003549396 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 67.50774946138796, 76.70202320961018 ], "I": [ 3.379327510519746, 8.897820124597144 ], "capability": [ 36.01781590414684, 42.01829735190088 ], "composite": { "A": [ 0.0550759146094291, 0.8570777384335173 ], "B": [ 0.03640384449772412, 0.6085773029152636 ], "C": [ 0.03824716292321466, 0.5951928739121649 ] }, "seed": 89 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 73.02134054125926, "capability": 39.53807694084099, "intelligence": 6.054813340422714 }, "per_type": { "choice": { "accuracy": 0.3253333333333333, "calibration": 70.70931250000002, "cc": 13.719769237327435, "cc_by_type": { "choice": 13.719769237327435 }, "cc_tiered": 13.009276888663784, "cc_tiered_open": 11.596876448581234, "cc_tiered_sealed": 14.421677328746334, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 13.719769237327435 }, "noul": { "accuracy": 0.17866666666666667, "calibration": 68.18437333333334, "cc": -64.26666666666667, "cc_by_type": { "noul": -64.26666666666667 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.312, "calibration": 80.17033579044444, "cc": 5.329298245614, "cc_by_type": { "score": 5.329298245614 }, "cc_tiered": 5.155163132604359, "cc_tiered_open": 6.303985276493753, "cc_tiered_sealed": 4.006340988714965, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 5.329298245614 } }, "raw_on_A": { "calibration": 75.9241654666809, "capability": 42.17966881905028, "intelligence": 8.435172171419651 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.23536198594902974, "Emphasis on Accuracy 60:20:20": 0.14175760897818884, "Emphasis on Cost 20:20:60": 0.3502184575801078, "Emphasis on Speed 20:60:20": 0.3525263048935194, "Intelligence only": 0.08878963459745969 }, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.28, "acc_among_decisive": 0.638095238095238, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "author submission" }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Small-EN", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (order 3643732b)", "underlying": "answerdotai/ModernBERT-base" }, { "addendum": null, "api_flag": true, "api_subset": true, "author": "Fastino", "axes": { "calibration": 68.83766854772081, "cost": 46.59842351525712, "intelligence": 6.439988814479404, "speed": 84.7052606179839 }, "calibration": { "parts": { "choice": { "basis": "provider_confidence_only", "ece_score": 37.74599383685215, "n": 121, "n_ece": 121, "score": 37.74599383685215, "top_label_ece": 0.31127003081573923 }, "noul": { "basis": "provider_confidence_only", "ece_score": 73.62788227580538, "n": 149, "n_ece": 149, "score": 73.62788227580538, "top_label_ece": 0.1318605886209731 }, "score": { "basis": "provider_confidence_only", "ece_score": 88.11046315790855, "n": 146, "n_ece": 146, "score": 88.11046315790855, "top_label_ece": 0.05944768421045723 } }, "score": 66.49477975685535 }, "capability": 37.63882868110011, "composite_ci95": { "A": [ 0.02337669748851963, 1.5368717897453439 ], "B": [ 0.015509495167746355, 1.1775764971931095 ], "C": [ 0.016233817700360857, 1.0672720762120442 ] }, "cost": { "basis": "ESTIMATE: published Fastino sources conflict (USD 0.03/M vs 0.15/M input); conservative higher USD 0.15/M input, 0/M output x measured tokens; no invoice", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.06026325 }, "display": "Fastino GLiNER-2.5-Decide (hosted API)", "endpoint_kind": "api", "full_coverage": true, "intelligence": { "I_open": 6.2445368778313695, "I_sealed": 11.055275519087195, "base": 8.649906198459282, "excess": -3.618526055358991, "gap": -4.810738641255825, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 18.73361063349411, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 18.270270270270274, "hard": 15.06763133060712, "judge": 24.851680949241928, "standard": 17.120133947258267 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -28.57142857142857, "hard": -21.739130434782606, "judge": 10.000000000000009, "standard": 11.111111111111116 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": -36.36363636363635, "hard": -28.70370370370372, "judge": -16.949152542372904, "standard": -99.99999999999996 } }, "sealed|choice": { "cc": 24.622840131922217, "n": { "easy": 21, "hard": 62, "judge": 42, "standard": 25 }, "tiers": { "easy": 15.859766277128548, "hard": 21.514629948364895, "judge": 31.09187749667111, "standard": 25.51724137931035 } }, "sealed|noul": { "cc": 8.542986425339368, "n": { "easy": 13, "hard": 25, "judge": 20, "standard": 17 }, "tiers": { "easy": 23.076923076923084, "hard": -19.999999999999996, "judge": 19.999999999999996, "standard": 41.176470588235304 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 13, "hard": 33, "judge": 10, "standard": 19 }, "tiers": { "easy": -43.47826086956523, "hard": -46.51162790697674, "judge": 13.953488372093037, "standard": -7.272727272727275 } } } }, "jevbench_score": 0.28381932562808126, "key": "fastino-gliner-2-5-decide", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 72, "ranked": true, "ranks": { "A": 72, "B": 71, "C": 72, "capability": 72 }, "scores": { "A": 0.28381932562808126, "B": 0.20104475263143454, "C": 0.1970967539083898 }, "speed": { "adjustment": "none (API)", "n": 124, "p50_s_adjusted": 0.4409300498664379, "p50_s_raw": 0.4409300498664379, "p95_s_adjusted": 0.7675456959754229, "p95_s_raw": 0.7675456959754229 }, "status": { "answered_ok": 584, "missing": 0, "rows": 600, "status": "complete" }, "support": { "choice": "confidence", "noul": "confidence", "score": "confidence" }, "v16": { "api_subset": true, "api_subset_tag": "A", "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 57.955384019443365, 75.09432840392678 ], "I": [ 2.6696780799307342, 12.058569972283067 ], "capability": [ 31.452820601821198, 42.559784367127904 ], "composite": { "A": [ 0.02337669748851963, 1.5368717897453439 ], "B": [ 0.015509495167746355, 1.1775764971931095 ], "C": [ 0.016233817700360857, 1.0672720762120442 ] }, "seed": 38 }, "ci95_basis": "bootstrap over the API system's items (n = A sealed items) AND over the equating offset (pool systems resampled with replacement, items resampled; B_offset = 200)", "ci95_n": 300, "complete": true, "equated": { "calibration": 68.83766854772081, "capability": 37.63882868110011, "intelligence": 6.439988814479404 }, "equating_flag": null, "equating_offset": { "C": 2.3428887908654588, "I": -2.209917383979878 }, "g_med_used": -1.1922125858968342, "lane": "api", "metric_basis": "A u P (sealed side = A, n=300)", "n_items": 600, "n_public": 300, "n_sealed_side": 300, "offset_ci95": { "C": [ 1.8692848511563234, 5.1754496498420055 ], "I": [ -4.708219277619042, -0.4755385207395287 ] }, "on_S": null, "per_type": { "choice": { "accuracy": 0.38, "calibration": 37.74599383685215, "cc": 21.121451289904538, "cc_by_type": { "choice": 21.121451289904538 }, "cc_tiered": 21.678225382708163, "cc_tiered_open": 18.73361063349411, "cc_tiered_sealed": 24.622840131922217, "low_n": false, "n": 300, "n_by_type": { "choice": 300 }, "score": 21.121451289904538 }, "noul": { "accuracy": 0.5133333333333333, "calibration": 73.62788227580538, "cc": 2.6666666666666616, "cc_by_type": { "noul": 2.6666666666666616 }, "cc_tiered": 4.271493212669684, "cc_tiered_open": 0.0, "cc_tiered_sealed": 8.542986425339368, "low_n": false, "n": 150, "n_by_type": { "noul": 150 }, "score": 2.6666666666666616 }, "score": { "accuracy": 0.2866666666666667, "calibration": 88.11046315790855, "cc": -30.19693654266962, "cc_by_type": { "score": -30.19693654266962 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 150, "n_by_type": { "score": 150 }, "score": 0.0 } }, "raw_on_A": { "calibration": 66.49477975685535, "capability": 37.572342977657314, "intelligence": 8.649906198459282 }, "set": "A+P" }, "validity": { "choice": { "invalid_rate": 0.036666666666666625, "n": 300 }, "noul": { "invalid_rate": 0.00666666666666671, "n": 150 }, "score": { "invalid_rate": 0.026666666666666616, "n": 150 } }, "views": { "Balanced 33:33:33": 0.229265149700654, "Emphasis on Accuracy 60:20:20": 0.14434801484515522, "Emphasis on Cost 20:20:60": 0.31125534240285524, "Emphasis on Speed 20:60:20": 0.3395830513268285, "Intelligence only": 0.09279354579929197 }, "class": "decision-api", "open": "no", "licence": "proprietary hosted API", "repo": "https://fastino.ai/", "endpoint_condition": "external hosted API (Fastino); received the 300-item API subset and the 300 public items", "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "model_pin": "fastino/GLiNER-2.5-Decide (requested model id)", "last_measured_on": "2026-10-03", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 1.0, "acc_among_decisive": 0.5167785234899329, "valid": 149 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": null } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Convai Innovations", "axes": { "calibration": 82.96996460403702, "cost": 82.52942826196218, "intelligence": 5.471270906587037, "speed": 70.81965680870755 }, "calibration": { "parts": { "choice": { "ece_hard": 0.08964031250000004, "ece_score": 82.07193749999999, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 82.07193749999999 }, "noul": { "abstention_rate": 0.9946666666666667, "brier": 0.24736279077333312, "ece": 0.09575093333333361, "n": 375, "score": 80.84981333333327 }, "score": { "interval90_coverage": 0.968, "interval90_width": 0.8166666666666665, "n": 375, "nrps": 0.15890220709111097, "score": 85.9881429787778, "top_ece": 0.060667466666666586 } }, "score": 82.96996460403702 }, "capability": 44.22061775531203, "composite_ci95": { "A": [ 0.03846019800643322, 0.5898651799347397 ], "B": [ 0.025332676441949146, 0.41662539701360535 ], "C": [ 0.02670847083780085, 0.40962859717690253 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: documented hosted-model estimate; no exact base-model floor applies. $0.01/M input, $0/M output; same-class hosted-encoder estimate; no exact-base market floor.", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.003822543103448276 }, "display": "Laya typed-decisions", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 5.6928563402143615, "I_sealed": 5.249685472959713, "base": 5.471270906587037, "excess": -2.130216775183595, "gap": 0.4431708672546488, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 10.771453572375108, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 6.5945945945945965, "hard": -1.9188424032714584, "judge": 24.851680949241928, "standard": 17.120133947258267 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -85.71428571428572, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "open|score": { "cc": 6.307115448267977, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 5.935454545454554, "hard": -3.607592592592601, "judge": 9.858813559322, "standard": 20.99481481481481 } }, "sealed|choice": { "cc": 7.102994687643811, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 5.550218017884884, "hard": 2.771493212669664, "judge": 6.890130353817524, "standard": 16.861682473210998 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -96.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "sealed|score": { "cc": 8.646061731235328, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 9.860354609929068, "hard": 2.728602941176461, "judge": 13.044251497005966, "standard": 13.276548223350238 } } } }, "jevbench_score": 0.21666109526429145, "key": "laya-typed-decisions", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 73, "ranked": true, "ranks": { "A": 73, "B": 73, "C": 73, "capability": 59 }, "scores": { "A": 0.21666109526429145, "B": 0.14825243435052415, "C": 0.1504590939335357 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 1.8777340129803632, "p50_s_raw": 0.8638670064901817, "p95_s_adjusted": 4.409610662701016, "p95_s_raw": 2.1298053313505076 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 77.7245419121787, 86.71414939090002 ], "I": [ 2.9916515598721545, 7.823617519507115 ], "capability": [ 41.079834277143355, 46.819056265617796 ], "composite": { "A": [ 0.03846019800643322, 0.5898651799347397 ], "B": [ 0.025332676441949146, 0.41662539701360535 ], "C": [ 0.02670847083780085, 0.40962859717690253 ] }, "seed": 58 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 82.96996460403702, "capability": 44.22061775531203, "intelligence": 5.471270906587037 }, "per_type": { "choice": { "accuracy": 0.2733333333333333, "calibration": 82.07193749999999, "cc": 7.069711925579947, "cc_by_type": { "choice": 7.069711925579947 }, "cc_tiered": 8.93722413000946, "cc_tiered_open": 10.771453572375108, "cc_tiered_sealed": 7.102994687643811, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 7.069711925579947 }, "noul": { "accuracy": 0.005333333333333333, "calibration": 80.84981333333327, "cc": -98.93333333333332, "cc_by_type": { "noul": -98.93333333333332 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.36533333333333334, "calibration": 85.9881429787778, "cc": 7.23879824561402, "cc_by_type": { "score": 7.23879824561402 }, "cc_tiered": 7.476588589751652, "cc_tiered_open": 6.307115448267977, "cc_tiered_sealed": 8.646061731235328, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 7.23879824561402 } }, "raw_on_A": { "calibration": 79.92527034876068, "capability": 42.6053892600956, "intelligence": 5.28550817143052 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.17186613033097944, "Emphasis on Accuracy 60:20:20": 0.10420153536931118, "Emphasis on Cost 20:20:60": 0.2566823909067557, "Emphasis on Speed 20:60:20": 0.2523472425383811, "Intelligence only": 0.06551257180569732 }, "class": "system-one-open", "open": "yes", "licence": "apache-2.0, not gated", "repo": "https://huggingface.co/convaiinnovations/laya/tree/1c5edc17a7acd8701df6fc341c0d179f1c62c982/typed-decisions", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU, Sandy (AMD Ryzen 5 3600), 4 of 12 threads, nice -n 5, shared host, offline (HF_HUB_OFFLINE=1)", "underlying": "convaiinnovations/laya @ 1c5edc17a7acd8701df6fc341c0d179f1c62c982, typed-decisions/; ModernBERT-large, max_len 1024, distinct head and calibration from the English root. Fresh full run on 29 Sep; in-process, offline, shared AMD Ryzen 5 3600 CPU, four threads. Measured latency includes shared-host conditions; it is not a controlled GPU comparison. Complete valid probabilities on all types; frequent noul abstention lowers Intelligence to its zero floor and thus the composite to zero. This is not a transport failure.", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.005333333333333333, "acc_among_decisive": 1.0, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Hemant (heman10x)", "axes": { "calibration": 75.69890703057408, "cost": 86.62313527343974, "intelligence": 5.035901218168153, "speed": 81.49274954261848 }, "calibration": { "parts": { "choice": { "ece_hard": 0.13155601347182505, "ece_score": 73.68879730563499, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 73.68879730563499 }, "noul": { "abstention_rate": 1.0, "brier": 0.2495580281877627, "ece": 0.12226520797197521, "n": 375, "score": 75.54695840560495 }, "score": { "interval90_coverage": 0.9893333333333333, "interval90_width": 0.916888888888889, "n": 375, "nrps": 0.17500942351683016, "score": 77.8609653804823, "top_ece": 0.1338856344367619 } }, "score": 75.69890703057408 }, "capability": 40.36740412437111, "composite_ci95": { "A": [ 0.0373899767734964, 0.4672540094953807 ], "B": [ 0.024598584677982253, 0.3262815362283655 ], "C": [ 0.025965261648261392, 0.3244819510384588 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.002791871921182266 }, "display": "openJev Verdict 1.4", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 5.214507036728864, "I_sealed": 4.857295399607441, "base": 5.035901218168153, "excess": -2.216176005316821, "gap": 0.3572116371214227, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 15.643521110186592, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": -5.081081081081076, "hard": 13.180245360176167, "judge": 24.851680949241928, "standard": 17.120133947258267 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 11.206824837702234, "hard": -8.9038551649125, "judge": -0.9107640863542654, "standard": -33.62275309182012 } }, "sealed|choice": { "cc": 8.257811980618104, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 5.550218017884884, "hard": 6.334841628959255, "judge": 9.683426443202995, "standard": 11.319127971425068 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "sealed|score": { "cc": 6.314074218204219, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 14.960976046364117, "hard": 5.800113224444569, "judge": 5.914716157590583, "standard": 3.617582382564022 } } } }, "jevbench_score": 0.17222636557331997, "key": "openjev-verdict-1.4", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 74, "ranked": true, "ranks": { "A": 74, "B": 75, "C": 74, "capability": 69 }, "scores": { "A": 0.17222636557331997, "B": 0.11682093815444984, "C": 0.11960164275925 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.7352308319881559, "p50_s_raw": 0.2926154159940779, "p95_s_adjusted": 0.9644980074837803, "p95_s_raw": 0.40724900374189016 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 70.20079626876355, 79.44394588474857 ], "I": [ 2.9611688986465445, 7.176652677169861 ], "capability": [ 37.175911743687855, 42.54492005899604 ], "composite": { "A": [ 0.0373899767734964, 0.4672540094953807 ], "B": [ 0.024598584677982253, 0.3262815362283655 ], "C": [ 0.025965261648261392, 0.3244819510384588 ] }, "seed": 83 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 75.69890703057408, "capability": 40.36740412437111, "intelligence": 5.035901218168153 }, "per_type": { "choice": { "accuracy": 0.2866666666666667, "calibration": 73.68879730563499, "cc": 8.774854826028024, "cc_by_type": { "choice": 8.774854826028024 }, "cc_tiered": 11.950666545402349, "cc_tiered_open": 15.643521110186592, "cc_tiered_sealed": 8.257811980618104, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 8.774854826028024 }, "noul": { "accuracy": 0.0, "calibration": 75.54695840560495, "cc": -100.0, "cc_by_type": { "noul": -100.0 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.344, "calibration": 77.8609653804823, "cc": 4.055284730748376, "cc_by_type": { "score": 4.055284730748376 }, "cc_tiered": 3.1570371091021094, "cc_tiered_open": 0.0, "cc_tiered_sealed": 6.314074218204219, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 4.055284730748376 } }, "raw_on_A": { "calibration": 73.35601823970862, "capability": 39.682189077168275, "intelligence": 6.008359914627925 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.13684263027905905, "Emphasis on Accuracy 60:20:20": 0.08186844689007573, "Emphasis on Cost 20:20:60": 0.2066197471065469, "Emphasis on Speed 20:60:20": 0.20540349835976665, "Intelligence only": 0.05108478843903597 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/heman10x/rlcd-modernbert-151m", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "underlying": "GLiClass ModernBERT-base fine-tuned decision model, 151M; v1.4 fixed inference engine", "adapter_id": "images/openjev-verdict-1.4/spec.json:warm_adapter.WarmLoaded", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.0, "acc_among_decisive": null, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Alibaba Qwen / neutral reproduction", "axes": { "calibration": 10.94062892769364, "cost": 77.58073177551512, "intelligence": 5.554864880982027, "speed": 93.52703208725202 }, "calibration": { "parts": { "choice": { "ece_hard": 0.6186890197483605, "ece_score": 0.0, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 0.0 }, "noul": { "abstention_rate": 0.0, "brier": 0.580040467448705, "ece": 0.5809468363625269, "n": 371, "score": 0.0 }, "score": { "interval90_coverage": 0.5067750677506775, "interval90_width": 0.31887985546522124, "n": 369, "nrps": 0.34356226433838155, "score": 32.82188678308092, "top_ece": 0.6266298915392519 } }, "score": 10.94062892769364 }, "capability": 8.247746904337834, "composite_ci95": { "A": [ 0.04648667178844407, 0.37042416292233377 ], "B": [ 0.033778521738855774, 0.30178355486696584 ], "C": [ 0.032282410964197275, 0.2572390020293985 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.005588676108374384 }, "display": "Raw Qwen3 0.6B direct logits", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 6.274289147536377, "I_sealed": 4.835440614427676, "base": 5.554864880982027, "excess": -1.1345391093295425, "gap": 1.4388485331087013, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 18.822867442609134, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 29.945945945945947, "hard": 9.405473419314262, "judge": 28.80685563612393, "standard": 17.120133947258267 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -57.14285714285714, "hard": -30.434782608695656, "judge": -30.000000000000004, "standard": -22.22222222222222 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 39.408700317953674, "hard": -23.213473292255358, "judge": -47.812096684203475, "standard": -39.963164880126655 } }, "sealed|choice": { "cc": 14.506321843283029, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 28.164954548813856, "hard": -1.8099547511312433, "judge": 26.443202979515842, "standard": 22.404236974996934 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -24.0, "hard": -22.330097087378643, "judge": -5.263157894736848, "standard": -1.4084507042253502 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -8.286778561594211, "hard": -56.317892789427006, "judge": -50.51708927969312, "standard": -75.44853557058413 } } } }, "jevbench_score": 0.1673536172919218, "key": "raw-qwen3-0.6b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 75, "ranked": true, "ranks": { "A": 75, "B": 74, "C": 75, "capability": 92 }, "scores": { "A": 0.1673536172919218, "B": 0.12991425235961862, "C": 0.11621778978605682 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2079640818759799, "p50_s_raw": 0.02898204093798995, "p95_s_adjusted": 0.21345604490488768, "p95_s_raw": 0.03172802245244384 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 10.514844290675109, 11.37896862687295 ], "I": [ 3.4338072843331555, 7.572845892568445 ], "capability": [ 7.166123426414619, 9.279855986833727 ], "composite": { "A": [ 0.04648667178844407, 0.37042416292233377 ], "B": [ 0.033778521738855774, 0.30178355486696584 ], "C": [ 0.032282410964197275, 0.2572390020293985 ] }, "seed": 94 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 10.94062892769364, "capability": 8.247746904337834, "intelligence": 5.554864880982027 }, "per_type": { "choice": { "accuracy": 0.336, "calibration": 0.0, "cc": 15.0838835576859, "cc_by_type": { "choice": 15.0838835576859 }, "cc_tiered": 16.66459464294608, "cc_tiered_open": 18.822867442609134, "cc_tiered_sealed": 14.506321843283029, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 15.0838835576859 }, "noul": { "accuracy": 0.41333333333333333, "calibration": 0.0, "cc": -17.333333333333336, "cc_by_type": { "noul": -17.333333333333336 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.256, "calibration": 32.82188678308092, "cc": -46.098542643497574, "cc_by_type": { "score": -46.098542643497574 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 10.73890139109323, "capability": 8.515857103448049, "intelligence": 6.292812815802869 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 0.181861744313326, "Emphasis on Accuracy 60:20:20": 0.10948842934445303, "Emphasis on Cost 20:20:60": 0.26903830305770593, "Emphasis on Speed 20:60:20": 0.27429427173292537, "Intelligence only": 0.06856152826445967 }, "class": "raw-logit-control", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/Qwen/Qwen3-0.6B", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen3-0.6B BF16", "adapter_id": "images/raw-qwen3-0.6b/spec.json:raw_adapter.RawControlAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 1.0, "acc_among_decisive": 0.41778975741239893, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Franck Verrot (franckverrot)", "axes": { "calibration": 81.86424479313416, "cost": 80.03797760430007, "intelligence": 4.827562367090417, "speed": 93.98230372494703 }, "calibration": { "parts": { "choice": { "ece_hard": 0.04337133550488594, "ece_score": 91.32573289902281, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 91.32573289902281 }, "noul": { "abstention_rate": 0.921832884097035, "brier": 0.2643957088948787, "ece": 0.12656064690026958, "n": 371, "score": 74.68787061994608 }, "score": { "interval90_coverage": 0.978319783197832, "interval90_width": 0.9372177055103887, "n": 369, "nrps": 0.17965857520325207, "score": 79.5791308604336, "top_ece": 0.11437940379403796 } }, "score": 81.86424479313416 }, "capability": 43.34590358011229, "composite_ci95": { "A": [ 0.036596960876541004, 0.39120491517125294 ], "B": [ 0.024012400167042817, 0.27027613731690037 ], "C": [ 0.02541455616426458, 0.2716700799800367 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.00462807881773399 }, "display": "lev-350m (Franck Verrot, LFM2.5-350M)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 4.690739555547254, "I_sealed": 4.964385178633581, "base": 4.827562367090417, "excess": -2.8470332655245705, "gap": -0.2736456230863267, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 14.072218666641763, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 10.486486486486491, "hard": 11.292859389745214, "judge": 16.941331575477918, "standard": 17.120133947258267 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -88.88888888888889 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 16.442424242424224, "hard": -5.066666666666664, "judge": 1.4508474576270802, "standard": -32.55555555555556 } }, "sealed|choice": { "cc": 14.893155535900744, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 21.513561451481806, "hard": 9.898190045248851, "judge": 14.338919925512121, "standard": 22.404236974996934 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -76.0, "hard": -92.23300970873787, "judge": -81.57894736842105, "standard": -94.36619718309859 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 15.455319148936175, "hard": -14.996568627450912, "judge": -3.196407185628769, "standard": -22.131472081218284 } } } }, "jevbench_score": 0.15377137469809216, "key": "lev-350m", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 76, "ranked": true, "ranks": { "A": 76, "B": 77, "C": 76, "capability": 63 }, "scores": { "A": 0.15377137469809216, "B": 0.1036628826651748, "C": 0.10678567687367513 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.1946538500036695, "p50_s_raw": 0.022326925001834752, "p95_s_adjusted": 0.2053556374099571, "p95_s_raw": 0.02767781870497856 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 75.71542881918619, 83.56232508492258 ], "I": [ 2.9345502485864183, 6.710083033760257 ], "capability": [ 40.03125074179408, 44.53569592988895 ], "composite": { "A": [ 0.036596960876541004, 0.39120491517125294 ], "B": [ 0.024012400167042817, 0.27027613731690037 ], "C": [ 0.02541455616426458, 0.2716700799800367 ] }, "seed": 60 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 81.86424479313416, "capability": 43.34590358011229, "intelligence": 4.827562367090417 }, "per_type": { "choice": { "accuracy": 0.3333333333333333, "calibration": 91.32573289902281, "cc": 14.74285497759628, "cc_by_type": { "choice": 14.74285497759628 }, "cc_tiered": 14.482687101271253, "cc_tiered_open": 14.072218666641763, "cc_tiered_sealed": 14.893155535900744, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 14.74285497759628 }, "noul": { "accuracy": 0.05333333333333334, "calibration": 74.68787061994608, "cc": -89.33333333333333, "cc_by_type": { "noul": -89.33333333333333 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.248, "calibration": 79.5791308604336, "cc": -8.448333333333302, "cc_by_type": { "score": -8.448333333333302 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 81.0219883791457, "capability": 43.49027409042074, "intelligence": 5.958559801695786 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 0.12144625080361596, "Emphasis on Accuracy 60:20:20": 0.07231333345467754, "Emphasis on Cost 20:20:60": 0.18259637420585254, "Emphasis on Speed 20:60:20": 0.1852875240477895, "Intelligence only": 0.04500322848109108 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 (code); weights under LiquidAI's LFM1.0 licence, following the LFM2.5-350M base", "repo": "https://github.com/franckverrot/lev", "gpu": "RTX6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container", "underlying": "LiquidAI/LFM2.5-350M with a LoRA and a 6.5M-parameter pointer head (a kev clone)", "adapter_id": "images/lev-350m/spec.json:lev_v15.LevAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-05", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.07816711590296496, "acc_among_decisive": 0.6896551724137931, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Alibaba Qwen / neutral reproduction", "axes": { "calibration": 11.516326812547591, "cost": 68.54983190559568, "intelligence": 5.124286554530066, "speed": 93.23530725162705 }, "calibration": { "parts": { "choice": { "ece_hard": 0.7144612706448337, "ece_score": 0.0, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 0.0 }, "noul": { "abstention_rate": 0.0026954177897574594, "brier": 0.5809971640519896, "ece": 0.5815255292937018, "n": 371, "score": 0.0 }, "score": { "interval90_coverage": 0.4959349593495935, "interval90_width": 0.23238482384823847, "n": 369, "nrps": 0.3090203912471445, "score": 34.548980437642776, "top_ece": 0.5704149904947305 } }, "score": 11.516326812547591 }, "capability": 8.320306683538828, "composite_ci95": { "A": [ 0.040682757177666354, 0.3224989584923113 ], "B": [ 0.029352394526125654, 0.25964193163917776 ], "C": [ 0.02825191470671274, 0.22395761006410508 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.011177352216748768 }, "display": "Raw Qwen3 1.7B direct logits", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 7.018912428422589, "I_sealed": 3.2296606806375423, "base": 5.124286554530066, "excess": 1.2158641053468031, "gap": 3.789251747785047, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 21.05673728526777, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 41.621621621621614, "hard": -0.03145643284050508, "judge": 24.851680949241928, "standard": 47.25826705734617 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -57.14285714285714, "hard": -30.434782608695656, "judge": -30.000000000000004, "standard": -22.22222222222222 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 16.019988364571248, "hard": -4.886927335935498, "judge": -32.17506026172301, "standard": -190.1002337898353 } }, "sealed|choice": { "cc": 9.688982041912627, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 21.513561451481806, "hard": -5.3733031674208345, "judge": 21.787709497206727, "standard": 15.753171572853812 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -24.0, "hard": -22.330097087378643, "judge": -5.263157894736848, "standard": -1.4084507042253502 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -19.19653010919571, "hard": -25.671423213473844, "judge": -56.434399118166525, "standard": -31.617468115507542 } } } }, "jevbench_score": 0.13671939226506033, "key": "raw-qwen3-1.7b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 77, "ranked": true, "ranks": { "A": 77, "B": 76, "C": 77, "capability": 91 }, "scores": { "A": 0.13671939226506033, "B": 0.10452214659357986, "C": 0.09494402240629189 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.21396372243762016, "p50_s_raw": 0.03198186121881008, "p95_s_adjusted": 0.221885605994612, "p95_s_raw": 0.03594280299730599 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 11.044864988748188, 11.991879282296361 ], "I": [ 3.263077550449943, 7.14487317844146 ], "capability": [ 7.338168404696409, 9.339947930556471 ], "composite": { "A": [ 0.040682757177666354, 0.3224989584923113 ], "B": [ 0.029352394526125654, 0.25964193163917776 ], "C": [ 0.02825191470671274, 0.22395761006410508 ] }, "seed": 95 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 11.516326812547591, "capability": 8.320306683538828, "intelligence": 5.124286554530066 }, "per_type": { "choice": { "accuracy": 0.30666666666666664, "calibration": 0.0, "cc": 11.332569176700131, "cc_by_type": { "choice": 11.332569176700131 }, "cc_tiered": 15.372859663590198, "cc_tiered_open": 21.05673728526777, "cc_tiered_sealed": 9.688982041912627, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 11.332569176700131 }, "noul": { "accuracy": 0.41333333333333333, "calibration": 0.0, "cc": -17.333333333333336, "cc_by_type": { "noul": -17.333333333333336 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.304, "calibration": 34.548980437642776, "cc": -31.4596497298246, "cc_by_type": { "score": -31.4596497298246 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 11.45520847967844, "capability": 9.226470190380851, "intelligence": 6.997731901083261 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 0.14292662698061412, "Emphasis on Accuracy 60:20:20": 0.08598558171809113, "Emphasis on Cost 20:20:60": 0.21037073826496294, "Emphasis on Speed 20:60:20": 0.21708826744822254, "Intelligence only": 0.05382204747082655 }, "class": "raw-logit-control", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/Qwen/Qwen3-1.7B", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen3-1.7B BF16", "adapter_id": "images/raw-qwen3-1.7b/spec.json:raw_adapter.RawControlAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.9973045822102425, "acc_among_decisive": 0.4189189189189189, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Jared Palmer", "axes": { "calibration": 61.20618525995795, "cost": 80.09398520305423, "intelligence": 4.598433868646836, "speed": 92.39904218127765 }, "calibration": { "parts": { "choice": { "ece_hard": 0.22209098434273397, "ece_score": 55.5818031314532, "mean_tvd": null, "n": 729, "n_ece_hard": 307, "n_tvd": 0, "score": 55.5818031314532 }, "noul": { "abstention_rate": 0.7439353099730458, "brier": 0.291543126684636, "ece": 0.1912938005390836, "n": 371, "score": 61.74123989218327 }, "score": { "interval90_coverage": 0.9239130434782609, "interval90_width": 0.835824275362319, "n": 368, "nrps": 0.21140059009020898, "score": 66.29551275623737, "top_ece": 0.23134457739252184 } }, "score": 61.20618525995795 }, "capability": 32.90230956430239, "composite_ci95": { "A": [ 0.024058315307534144, 0.35808460603860887 ], "B": [ 0.015784497925561473, 0.2496641339569836 ], "C": [ 0.016707163408009824, 0.2486698653045895 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.004608226600985222 }, "display": "kev 0.5B", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 4.849742544692461, "I_sealed": 4.3471251926012116, "base": 4.598433868646836, "excess": -2.0707702903469944, "gap": 0.5026173520912494, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 14.549227634077383, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 6.5945945945945965, "hard": 11.292859389745214, "judge": 24.851680949241928, "standard": 9.585600669736289 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -85.71428571428572, "hard": -82.6086956521739, "judge": -100.0, "standard": -66.66666666666667 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 17.000794018795805, "hard": -8.594117745107855, "judge": -32.75818090283609, "standard": -52.88804065591748 } }, "sealed|choice": { "cc": 13.041375577803636, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 17.522725593082576, "hard": 7.352941176470571, "judge": 18.06331471135942, "standard": 14.644660672496622 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -48.0, "hard": -78.64077669902912, "judge": -71.05263157894737, "standard": -66.19718309859155 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 4.208590362582387, "hard": -22.033232980160843, "judge": -8.39299678470844, "standard": -23.154804312918653 } } } }, "jevbench_score": 0.131588623265787, "key": "kev-0.5b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 78, "ranked": true, "ranks": { "A": 78, "B": 78, "C": 78, "capability": 78 }, "scores": { "A": 0.131588623265787, "B": 0.08911346438551963, "C": 0.09138098837901876 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2379663866944611, "p50_s_raw": 0.043983193347230554, "p95_s_adjusted": 0.24186897580511868, "p95_s_raw": 0.04593448790255934 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 55.41312884823437, 65.28644013850979 ], "I": [ 2.5515112413087477, 6.5563351690373795 ], "capability": [ 29.53128001342342, 35.43777813183522 ], "composite": { "A": [ 0.024058315307534144, 0.35808460603860887 ], "B": [ 0.015784497925561473, 0.2496641339569836 ], "C": [ 0.016707163408009824, 0.2486698653045895 ] }, "seed": 52 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 61.20618525995795, "capability": 32.90230956430239, "intelligence": 4.598433868646836 }, "per_type": { "choice": { "accuracy": 0.31866666666666665, "calibration": 55.5818031314532, "cc": 12.8671977871034, "cc_by_type": { "choice": 12.8671977871034 }, "cc_tiered": 13.795301605940509, "cc_tiered_open": 14.549227634077383, "cc_tiered_sealed": 13.041375577803636, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 12.8671977871034 }, "noul": { "accuracy": 0.14133333333333334, "calibration": 61.74123989218327, "cc": -71.73333333333333, "cc_by_type": { "noul": -71.73333333333333 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.272, "calibration": 66.29551275623737, "cc": -15.865897379211603, "cc_by_type": { "score": -15.865897379211603 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 65.8831405050141, "capability": 35.92446865234692, "intelligence": 5.965796799679742 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.028000000000000025, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.01866666666666672, "n": 375 } }, "views": { "Balanced 33:33:33": 0.10538840072414271, "Emphasis on Accuracy 60:20:20": 0.06258833605506466, "Emphasis on Cost 20:20:60": 0.15914261116628434, "Emphasis on Speed 20:60:20": 0.1611592968234279, "Intelligence only": 0.03889464633042005 }, "class": "jev-rebuild", "open": true, "licence": "Apache-2.0", "repo": "https://github.com/jaredpalmer/kev", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen2.5-0.5B + LoRA + learned pointer head; jaredpalmer/kev-0.5b", "adapter_id": "images/kev-0.5b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.2560646900269542, "acc_among_decisive": 0.5578947368421052, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Hemant (heman10x)", "axes": { "calibration": 49.11476555745841, "cost": 86.62313527343974, "intelligence": 4.1496399816416485, "speed": 79.40046783928 }, "calibration": { "parts": { "choice": { "ece_hard": 0.3268574231249395, "ece_score": 34.6285153750121, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 34.6285153750121 }, "noul": { "abstention_rate": 0.6666666666666667, "brier": 0.28308617598601715, "ece": 0.20554105202175244, "n": 375, "score": 58.89178959564951 }, "score": { "interval90_coverage": 0.816, "interval90_width": 0.6600000000000001, "n": 375, "nrps": 0.21771137902642015, "score": 53.823991701713624, "top_ece": 0.3529043934696537 } }, "score": 49.11476555745841 }, "capability": 26.63220276955003, "composite_ci95": { "A": [ 0.016680739292552642, 0.27473431836523593 ], "B": [ 0.010947780723182459, 0.1924832136378699 ], "C": [ 0.011583846730939337, 0.19078772108696943 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.002791871921182266 }, "display": "openJev Verdict (heman10x, ModernBERT-base 151M)", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 5.710480975731317, "I_sealed": 2.5887989875519795, "base": 4.1496399816416485, "excess": 0.5482943457410934, "gap": 3.121681988179337, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 17.13144292719395, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 6.5945945945945965, "hard": 13.180245360176167, "judge": 20.896506262359924, "standard": 24.654667224780237 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -85.71428571428572, "hard": -56.52173913043479, "judge": -80.0, "standard": -55.55555555555556 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 8.056046691320095, "hard": -27.078088834020786, "judge": -9.574770356665475, "standard": -54.57566450814868 } }, "sealed|choice": { "cc": 7.766396962655938, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 10.87133249575052, "hard": 5.316742081447947, "judge": 6.890130353817524, "standard": 12.427638871782255 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -64.0, "hard": -49.51456310679612, "judge": -63.1578947368421, "standard": -60.56338028169015 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 11.069847893733552, "hard": -5.166740337464448, "judge": -15.492495207027046, "standard": -16.23629733590537 } } } }, "jevbench_score": 0.09650709186186587, "key": "openjev-verdict", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 79, "ranked": true, "ranks": { "A": 79, "B": 79, "C": 79, "capability": 86 }, "scores": { "A": 0.09650709186186587, "B": 0.06541514759096348, "C": 0.0670188137929624 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.6885442301630974, "p50_s_raw": 0.2692721150815487, "p95_s_adjusted": 1.66732926396653, "p95_s_raw": 0.7586646319832651 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 43.627592951745086, 53.02973228229396 ], "I": [ 2.2588266004870072, 6.027430783705419 ], "capability": [ 23.293166910490978, 29.139107914545924 ], "composite": { "A": [ 0.016680739292552642, 0.27473431836523593 ], "B": [ 0.010947780723182459, 0.1924832136378699 ], "C": [ 0.011583846730939337, 0.19078772108696943 ] }, "seed": 82 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 49.11476555745841, "capability": 26.63220276955003, "intelligence": 4.1496399816416485 }, "per_type": { "choice": { "accuracy": 0.2906666666666667, "calibration": 34.6285153750121, "cc": 9.286397696162448, "cc_by_type": { "choice": 9.286397696162448 }, "cc_tiered": 12.448919944924945, "cc_tiered_open": 17.13144292719395, "cc_tiered_sealed": 7.766396962655938, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 9.286397696162448 }, "noul": { "accuracy": 0.2, "calibration": 58.89178959564951, "cc": -60.0, "cc_by_type": { "noul": -60.0 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.2693333333333333, "calibration": 53.823991701713624, "cc": -9.675453706526316, "cc_by_type": { "score": -9.675453706526316 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 49.678974901443674, "capability": 27.759290067036318, "intelligence": 5.839605232628962 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.07793885393113759, "Emphasis on Accuracy 60:20:20": 0.046097376471726674, "Emphasis on Cost 20:20:60": 0.11949201576637343, "Emphasis on Speed 20:60:20": 0.11862755697857134, "Intelligence only": 0.028581910146043114 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0", "repo": "https://github.com/Heman10x-NGU/openJev-verdict-2.0", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "underlying": "GLiClass ModernBERT-base (knowledgator/gliclass-modern-base-v2.0) fine-tuned, 151M", "adapter_id": "images/openjev-verdict/spec.json:warm_adapter.WarmLoaded", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.3333333333333333, "acc_among_decisive": 0.6, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Chinh Nguyen (ncchinh)", "axes": { "calibration": 60.96568104722222, "cost": 79.56261245432609, "intelligence": 4.019045994532796, "speed": 94.91846454857637 }, "calibration": { "parts": { "choice": { "ece_hard": 0.22390249999999998, "ece_score": 55.2195, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 55.2195 }, "noul": { "abstention_rate": 0.712, "brier": 0.28332537661333335, "ece": 0.2195469333333333, "n": 375, "score": 56.09061333333334 }, "score": { "interval90_coverage": 0.9253333333333333, "interval90_width": 0.8177777777777777, "n": 375, "nrps": 0.1894891371666667, "score": 71.58692980833334, "top_ece": 0.18938613333333326 } }, "score": 60.96568104722222 }, "capability": 32.492363520877504, "composite_ci95": { "A": [ 0.020959672741331675, 0.3032118958147926 ], "B": [ 0.01370004084448916, 0.2103085758125444 ], "C": [ 0.014555328292591438, 0.21056381653805042 ] }, "cost": { "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): DeepInfra same-size encoders (bge-large, e5-large, Qwen3-Embedding-0.6B) $0.01/M input (SEA-LION-ModernBERT-300M has no market listing; the ModernBERT-large-class reference of the Laya/Certo/OpenDecision rows) x the package's own measured usage.input_tokens; one forward pass, nothing generated; run in process on 1x H100 80GB (Lium), quyet 1.0.0; estimated, not charged", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.004800055418719211 }, "display": "Quyet-1.0-Small (Chinh Nguyen, SEA-LION-ModernBERT-300M encoder with fixed heads)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 4.071843176603614, "I_sealed": 3.966248812461978, "base": 4.019045994532796, "excess": -2.467793278296608, "gap": 0.10559436414163592, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 12.215529529810842, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 2.702702702702708, "hard": 5.630701478452354, "judge": 20.896506262359924, "standard": 17.120133947258267 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -71.42857142857143, "hard": -56.52173913043479, "judge": -40.0, "standard": -55.55555555555556 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 13.416060606060597, "hard": -26.448333333333316, "judge": -3.2088135593220546, "standard": -76.01518518518517 } }, "sealed|choice": { "cc": 11.898746437385935, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 17.522725593082576, "hard": 5.316742081447947, "judge": 19.92551210428307, "standard": 10.210617071067881 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -72.0, "hard": -59.22330097087378, "judge": -65.78947368421053, "standard": -46.47887323943662 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 20.691063829787236, "hard": -7.6461764705882285, "judge": -0.1282634730539245, "standard": 0.04786802030454229 } } } }, "jevbench_score": 0.08963716950381444, "key": "quyet-1-0-small", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 80, "ranked": true, "ranks": { "A": 80, "B": 80, "C": 80, "capability": 80 }, "scores": { "A": 0.08963716950381444, "B": 0.0601437708059232, "C": 0.06224803437764891 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.17903870801674202, "p50_s_raw": 0.01451935400837101, "p95_s_adjusted": 0.17997269080078693, "p95_s_raw": 0.014986345400393473 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 55.688303752892594, 64.99464873956296 ], "I": [ 2.4305093708220933, 6.1818340512057315 ], "capability": [ 29.486356353832495, 35.15105810056002 ], "composite": { "A": [ 0.020959672741331675, 0.3032118958147926 ], "B": [ 0.01370004084448916, 0.2103085758125444 ], "C": [ 0.014555328292591438, 0.21056381653805042 ] }, "seed": 88 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 60.96568104722222, "capability": 32.492363520877504, "intelligence": 4.019045994532796 }, "per_type": { "choice": { "accuracy": 0.30533333333333335, "calibration": 55.2195, "cc": 11.162054886655328, "cc_by_type": { "choice": 11.162054886655328 }, "cc_tiered": 12.057137983598388, "cc_tiered_open": 12.215529529810842, "cc_tiered_sealed": 11.898746437385935, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 11.162054886655328 }, "noul": { "accuracy": 0.20533333333333334, "calibration": 56.09061333333334, "cc": -58.93333333333332, "cc_by_type": { "noul": -58.93333333333332 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.2986666666666667, "calibration": 71.58692980833334, "cc": -4.271587719298253, "cc_by_type": { "score": -4.271587719298253 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 55.62018953114671, "capability": 29.685347509548446, "intelligence": 3.750505487950187 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.07128318534334778, "Emphasis on Accuracy 60:20:20": 0.04197968509922768, "Emphasis on Cost 20:20:60": 0.10875180747478416, "Emphasis on Speed 20:60:20": 0.11026128891416027, "Intelligence only": 0.025967427058159954 }, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.288, "acc_among_decisive": 0.7129629629629629, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "author submission" }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Small", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (order 3643732b)", "underlying": "aisingapore/SEA-LION-ModernBERT-300M" }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Chinh Nguyen (ncchinh)", "axes": { "calibration": 58.46458026762962, "cost": 88.43590631389968, "intelligence": 3.6964771541368573, "speed": 95.09610910731021 }, "calibration": { "parts": { "choice": { "ece_hard": 0.2315684375, "ece_score": 53.6863125, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 53.6863125 }, "noul": { "abstention_rate": 0.8026666666666666, "brier": 0.28768576031999976, "ece": 0.19567466666666677, "n": 375, "score": 60.86506666666664 }, "score": { "interval90_coverage": 0.8933333333333333, "interval90_width": 0.7453333333333331, "n": 375, "nrps": 0.1976125006088888, "score": 60.84236163622223, "top_ece": 0.2927701333333333 } }, "score": 58.46458026762962 }, "capability": 31.08052871088324, "composite_ci95": { "A": [ 0.019766287637893492, 0.2734027471253872 ], "B": [ 0.012911567780248491, 0.18895613284128507 ], "C": [ 0.013726588637426037, 0.1898630188370744 ] }, "cost": { "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): DeepInfra base-size encoders (bge-base, e5-base, gte-base, all-mpnet-base) $0.005/M input (mmBERT-small 16-layer, 183M; base-size class reference, errs high vs the small-encoder $0.004 class) x the package's own measured usage.input_tokens; one forward pass, nothing generated; run in process on 1x H100 80GB (Lium), quyet 1.0.0; estimated, not charged", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0024292364532019703 }, "display": "Quyet-1.0-Tiny (Chinh Nguyen, mmBERT-small encoder (16 layers kept) with fixed heads)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 1.2549826648038989, "I_sealed": 6.1379716434698155, "base": 3.6964771541368573, "excess": -7.456376621104161, "gap": -4.882988978665916, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 3.764947994411697, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": -12.86486486486486, "hard": 5.630701478452354, "judge": 12.986156888595913, "standard": -5.4834658853076625 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -71.42857142857143, "hard": -56.52173913043479, "judge": -80.0, "standard": -88.88888888888889 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": -8.307575757575748, "hard": -21.650555555555574, "judge": -15.678474576271228, "standard": -51.14888888888891 } }, "sealed|choice": { "cc": 16.468453763405932, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 12.201611115216933, "hard": 14.479638009049756, "judge": 18.06331471135942, "standard": 20.18721517428256 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -84.0, "hard": -68.93203883495146, "judge": -71.05263157894737, "standard": -83.09859154929578 } }, "sealed|score": { "cc": 1.9454611670035158, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 16.69851063829787, "hard": -2.889509803921597, "judge": -1.9068263473054348, "standard": 10.017309644669991 } } } }, "jevbench_score": 0.07064766318734213, "key": "quyet-1-0-tiny", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 81, "ranked": true, "ranks": { "A": 81, "B": 81, "C": 81, "capability": 82 }, "scores": { "A": 0.07064766318734213, "B": 0.04711839739648976, "C": 0.04906087721343202 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.1753336349996971, "p50_s_raw": 0.012666817499848548, "p95_s_adjusted": 0.17641026510682423, "p95_s_raw": 0.013205132553412113 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 52.6105536767426, 62.31255872829814 ], "I": [ 2.382388771227772, 5.956289831309647 ], "capability": [ 27.937688273976466, 33.75728623357733 ], "composite": { "A": [ 0.019766287637893492, 0.2734027471253872 ], "B": [ 0.012911567780248491, 0.18895613284128507 ], "C": [ 0.013726588637426037, 0.1898630188370744 ] }, "seed": 90 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 58.46458026762962, "capability": 31.08052871088324, "intelligence": 3.6964771541368573 }, "per_type": { "choice": { "accuracy": 0.32133333333333336, "calibration": 53.6863125, "cc": 13.20822636719302, "cc_by_type": { "choice": 13.20822636719302 }, "cc_tiered": 10.116700878908814, "cc_tiered_open": 3.764947994411697, "cc_tiered_sealed": 16.468453763405932, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 13.20822636719302 }, "noul": { "accuracy": 0.12533333333333332, "calibration": 60.86506666666664, "cc": -74.93333333333334, "cc_by_type": { "noul": -74.93333333333334 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.33866666666666667, "calibration": 60.84236163622223, "cc": -1.831508771929835, "cc_by_type": { "score": -1.831508771929835 }, "cc_tiered": 0.9727305835017579, "cc_tiered_open": 0.0, "cc_tiered_sealed": 1.9454611670035158, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 54.460514049074064, "capability": 29.223962167243283, "intelligence": 3.987410285412503 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.05608574548219273, "Emphasis on Accuracy 60:20:20": 0.03279057174059279, "Emphasis on Cost 20:20:60": 0.08676445199059092, "Emphasis on Speed 20:60:20": 0.08720297577636826, "Intelligence only": 0.020203381773039073 }, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.19733333333333333, "acc_among_decisive": 0.6351351351351351, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "author submission" }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Tiny", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (order 3643732b)", "underlying": "jhu-clsp/mmBERT-small" }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Kotoba Labs", "axes": { "calibration": 67.96597988813339, "cost": 77.5922354035203, "intelligence": 2.8293414417857914, "speed": 67.62138193768119 }, "calibration": { "parts": { "choice": { "ece_hard": 0.20927949454905329, "ece_score": 58.14410109018935, "mean_tvd": null, "n": 682, "n_ece_hard": 276, "n_tvd": 0, "score": 58.14410109018935 }, "noul": { "abstention_rate": 0.848, "brier": 0.2599137925955484, "ece": 0.20243010437488554, "n": 375, "score": 59.5139791250229 }, "score": { "interval90_coverage": 0.8945945945945946, "interval90_width": 0.7720720720720718, "n": 370, "nrps": 0.16620611847475783, "score": 86.23985944918795, "top_ece": 0.05449834627074163 } }, "score": 67.96597988813339 }, "capability": 35.39766066495959, "composite_ci95": { "A": [ 0.0016094499426836406, 0.16453983269325992 ], "B": [ 0.0010275985408014102, 0.11262740891592737 ], "C": [ 0.0011176735713080835, 0.11426377270365268 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.005583743842364532 }, "display": "open-jev-deberta-v3-large (local CPU)", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 3.3193692126143426, "I_sealed": 2.3393136709572406, "base": 2.8293414417857914, "excess": -1.5933321007811418, "gap": 0.980055541657102, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 9.958107637843028, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 22.16216216216216, "hard": 1.855929537590448, "judge": 16.941331575477918, "standard": 9.585600669736289 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -71.42857142857143, "hard": -73.91304347826086, "judge": -70.0, "standard": -77.77777777777779 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": -3.536711678360427, "hard": -10.998114736543751, "judge": 10.982108090893682, "standard": -6.176287708459061 } }, "sealed|choice": { "cc": 5.620402697655405, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 17.522725593082576, "hard": -1.3009049773755852, "judge": 7.8212290502793484, "standard": 10.210617071067881 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -68.0, "hard": -88.3495145631068, "judge": -84.21052631578947, "standard": -91.54929577464789 } }, "sealed|score": { "cc": 1.3975383152163166, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -2.206062525510788, "hard": -5.817775736909847, "judge": 9.848256453782477, "standard": 4.953889631982955 } } } }, "jevbench_score": 0.03235814777127837, "key": "open-jev-deberta-v3-large", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 82, "ranked": true, "ranks": { "A": 82, "B": 82, "C": 82, "capability": 75 }, "scores": { "A": 0.03235814777127837, "B": 0.021367993730971636, "C": 0.02247093595227665 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 3.5280169868841766, "p50_s_raw": 1.6890084934420884, "p95_s_adjusted": 4.901524059008808, "p95_s_raw": 2.3757620295044037 }, "status": { "answered_ok": 1427, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 62.83136705170831, 72.03960745264857 ], "I": [ 1.0164743841847181, 4.99302167761034 ], "capability": [ 32.41699387669788, 37.92156574534849 ], "composite": { "A": [ 0.0016094499426836406, 0.16453983269325992 ], "B": [ 0.0010275985408014102, 0.11262740891592737 ], "C": [ 0.0011176735713080835, 0.11426377270365268 ] }, "seed": 75 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 67.96597988813339, "capability": 35.39766066495959, "intelligence": 2.8293414417857914 }, "per_type": { "choice": { "accuracy": 0.272, "calibration": 58.14410109018935, "cc": 6.899197635535144, "cc_by_type": { "choice": 6.899197635535144 }, "cc_tiered": 7.789255167749216, "cc_tiered_open": 9.958107637843028, "cc_tiered_sealed": 5.620402697655405, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 6.899197635535144 }, "noul": { "accuracy": 0.088, "calibration": 59.5139791250229, "cc": -82.4, "cc_by_type": { "noul": -82.4 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.31733333333333336, "calibration": 86.23985944918795, "cc": -0.7784697361159942, "cc_by_type": { "score": -0.7784697361159942 }, "cc_tiered": 0.6987691576081583, "cc_tiered_open": 0.0, "cc_tiered_sealed": 1.3975383152163166, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 68.63668964823074, "capability": 36.14242333704565, "intelligence": 3.64815702586055 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.09066666666666667, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.013333333333333308, "n": 375 } }, "views": { "Balanced 33:33:33": 0.0252055189654814, "Emphasis on Accuracy 60:20:20": 0.014715479135725748, "Emphasis on Cost 20:20:60": 0.0393479951952157, "Emphasis on Speed 20:60:20": 0.038983856596301915, "Intelligence only": 0.00905974708046916 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 (model card); DeBERTa-v3 keeps its own terms", "repo": "https://github.com/kotoba-lang/typed-decisions", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "underlying": "microsoft/deberta-v3-large", "adapter_id": "images/open-jev-deberta-v3-large/spec.json:warm_adapter.WarmLoaded", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.152, "acc_among_decisive": 0.5789473684210527, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Manavarya09 (Manav)", "axes": { "calibration": 72.92000413192591, "cost": 100.0, "intelligence": 2.3417095945172277, "speed": 89.48731594094681 }, "calibration": { "parts": { "choice": { "ece_hard": 0.13790718750000003, "ece_score": 72.4185625, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 72.4185625 }, "noul": { "abstention_rate": 0.5546666666666666, "brier": 0.28117829917333326, "ece": 0.18824453333333335, "n": 375, "score": 62.35109333333333 }, "score": { "interval90_coverage": 0.984, "interval90_width": 0.9684444444444443, "n": 375, "nrps": 0.18136406875111122, "score": 83.99035656244443, "top_ece": 0.06941440000000007 } }, "score": 72.92000413192591 }, "capability": 37.63085686322157, "composite_ci95": { "A": [ 0.00034752623458969486, 0.12857270817562724 ], "B": [ 0.00021950048318305755, 0.08623222919712645 ], "C": [ 0.0002413376629095103, 0.08928660289974111 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.000866152709359606 }, "display": "verdict-small (Manavarya09, multilingual-e5-small 118M)", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 4.005009650747796, "I_sealed": 0.6784095382866591, "base": 2.3417095945172277, "excess": 0.7532124700228935, "gap": 3.3266001124611373, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 12.01502895224339, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": -8.972972972972968, "hard": 13.180245360176167, "judge": 9.03098220171391, "standard": 24.654667224780237 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -28.57142857142857, "hard": -39.13043478260869, "judge": -40.0, "standard": -77.77777777777779 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 6.291818181818199, "hard": -6.231481481481493, "judge": 7.341694915254216, "standard": -11.9677777777778 } }, "sealed|choice": { "cc": 2.035228614859977, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": -7.75256817677922, "hard": -4.355203619909522, "judge": 6.890130353817524, "standard": 12.427638871782255 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -31.999999999999996, "hard": -45.63106796116505, "judge": -47.36842105263158, "standard": -46.47887323943662 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -0.7665248226950139, "hard": -6.760098039215645, "judge": 6.645688622754465, "standard": -2.297512690355341 } } } }, "jevbench_score": 0.01899384266854915, "key": "verdict-small", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 83, "ranked": true, "ranks": { "A": 83, "B": 83, "C": 83, "capability": 73 }, "scores": { "A": 0.01899384266854915, "B": 0.012337048142708213, "C": 0.013190168519825799 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.24820413775742053, "p50_s_raw": 0.04910206887871027, "p95_s_adjusted": 0.4533769000321627, "p95_s_raw": 0.15168845001608133 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 67.28225057545556, 76.20018850659073 ], "I": [ 0.6053648936331004, 4.532806367679299 ], "capability": [ 34.685562893137785, 39.80647930109692 ], "composite": { "A": [ 0.00034752623458969486, 0.12857270817562724 ], "B": [ 0.00021950048318305755, 0.08623222919712645 ], "C": [ 0.0002413376629095103, 0.08928660289974111 ] }, "seed": 108 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 72.92000413192591, "capability": 37.63085686322157, "intelligence": 2.3417095945172277 }, "per_type": { "choice": { "accuracy": 0.23733333333333334, "calibration": 72.4185625, "cc": 2.4658260943701484, "cc_by_type": { "choice": 2.4658260943701484 }, "cc_tiered": 7.0251287835516845, "cc_tiered_open": 12.01502895224339, "cc_tiered_sealed": 2.035228614859977, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 2.4658260943701484 }, "noul": { "accuracy": 0.2773333333333333, "calibration": 62.35109333333333, "cc": -44.53333333333334, "cc_by_type": { "noul": -44.53333333333334 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.2986666666666667, "calibration": 83.99035656244443, "cc": -2.2494035087719633, "cc_by_type": { "score": -2.2494035087719633 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 74.70758393102564, "capability": 38.932253022553525, "intelligence": 3.156922114081418 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.014681237741460948, "Emphasis on Accuracy 60:20:20": 0.008421478349576572, "Emphasis on Cost 20:20:60": 0.02342353351655187, "Emphasis on Speed 20:60:20": 0.02330657899907492, "Intelligence only": 0.005136403075864165 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 (verdictml code and the Manav2op/verdict-small checkpoint over intfloat/multilingual-e5-small)", "repo": "https://github.com/Manavarya09/verdict", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "underlying": "intfloat/multilingual-e5-small (118M multilingual bi-encoder) fine-tuned on a typed-decision mix; each option is scored against the rendered state by cosine similarity, with temperature scaling and a conformal abstain set on top. One encoder pass per option, no tokens generated.", "adapter_id": "images/verdict-small/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.44533333333333336, "acc_among_decisive": 0.6227544910179641, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Convai Innovations", "axes": { "calibration": 63.22326564159259, "cost": 84.89402784895724, "intelligence": 1.7910341507793501, "speed": 73.26302890816496 }, "calibration": { "parts": { "choice": { "ece_hard": 0.21969249999999993, "ece_score": 56.06150000000001, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 56.06150000000001 }, "noul": { "abstention_rate": 0.808, "brier": 0.28256342088000014, "ece": 0.18899546666666664, "n": 375, "score": 62.20090666666667 }, "score": { "interval90_coverage": 0.8826666666666667, "interval90_width": 0.7128888888888881, "n": 375, "nrps": 0.18963512817111114, "score": 71.4073902581111, "top_ece": 0.19110853333333327 } }, "score": 63.22326564159259 }, "capability": 32.50714989618597, "composite_ci95": { "A": [ 0.00020217712264809318, 0.06712460770140848 ], "B": [ 0.00012766304420524027, 0.044955197581294834 ], "C": [ 0.00014040077961673136, 0.046614310903755875 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0031881034482758625 }, "display": "Laya (Convai Innovations, ModernBERT-large 421M)", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 1.8026531464810367, "I_sealed": 1.7794151550776638, "base": 1.7910341507793501, "excess": -2.5501496510348707, "gap": 0.023237991403372904, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 5.40795943944311, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 14.37837837837838, "hard": -3.8062283737024116, "judge": 16.941331575477918, "standard": 2.051067392214312 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -57.14285714285714, "hard": -91.30434782608697, "judge": -80.0, "standard": -88.88888888888889 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 14.020606060606056, "hard": -11.245833333333334, "judge": -16.895084745762734, "standard": -50.04185185185186 } }, "sealed|choice": { "cc": 5.338245465232991, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 1.5593821594856503, "hard": -3.846153846153868, "judge": 9.683426443202995, "standard": 19.078704273925375 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -56.00000000000001, "hard": -84.46601941747574, "judge": -76.31578947368422, "standard": -74.64788732394366 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 4.452695035460996, "hard": -11.828823529411746, "judge": -8.140898203592851, "standard": -0.2669035532995023 } } } }, "jevbench_score": 0.008560099345041717, "key": "laya", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 84, "ranked": true, "ranks": { "A": 84, "B": 84, "C": 84, "capability": 79 }, "scores": { "A": 0.008560099345041717, "B": 0.005540634651880784, "C": 0.005944513434056748 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 1.800616904720664, "p50_s_raw": 0.825308452360332, "p95_s_adjusted": 2.619846137147397, "p95_s_raw": 1.2349230685736985 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 57.988437430874065, 67.35373438390093 ], "I": [ 0.5052598551452469, 3.643432797894196 ], "capability": [ 29.732944854873807, 34.91391985018961 ], "composite": { "A": [ 0.00020217712264809318, 0.06712460770140848 ], "B": [ 0.00012766304420524027, 0.044955197581294834 ], "C": [ 0.00014040077961673136, 0.046614310903755875 ] }, "seed": 56 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 63.22326564159259, "capability": 32.50714989618597, "intelligence": 1.7910341507793501 }, "per_type": { "choice": { "accuracy": 0.25466666666666665, "calibration": 56.06150000000001, "cc": 4.682511864952642, "cc_by_type": { "choice": 4.682511864952642 }, "cc_tiered": 5.373102452338051, "cc_tiered_open": 5.40795943944311, "cc_tiered_sealed": 5.338245465232991, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 4.682511864952642 }, "noul": { "accuracy": 0.11733333333333333, "calibration": 62.20090666666667, "cc": -76.53333333333333, "cc_by_type": { "noul": -76.53333333333333 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.2826666666666667, "calibration": 71.4073902581111, "cc": -7.6405701754385325, "cc_by_type": { "score": -7.6405701754385325 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 62.99195454673789, "capability": 33.098085372560654, "intelligence": 3.20421619838342 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.006594024627233607, "Emphasis on Accuracy 60:20:20": 0.0037729124634726325, "Emphasis on Cost 20:20:60": 0.010563725142022983, "Emphasis on Speed 20:60:20": 0.010499068205592536, "Intelligence only": 0.0022981141246738437 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/convaiinnovations/laya", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "underlying": "ModernBERT-large encoder + option-marker decision head, 421M, RLCD-trained", "adapter_id": "images/laya/spec.json:warm_adapter.WarmLoaded", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.192, "acc_among_decisive": 0.6111111111111112, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Mixedbread", "axes": { "calibration": 86.60772459153094, "cost": 60.33589314238483, "intelligence": 1.140707327109041, "speed": 92.58758170313658 }, "calibration": { "parts": { "choice": { "ece_hard": 0.06855480504522303, "ece_score": 86.28903899095539, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 86.28903899095539 }, "noul": { "abstention_rate": 1.0, "brier": 0.25057053855990563, "ece": 0.07454411727378144, "n": 375, "score": 85.09117654524371 }, "score": { "interval90_coverage": 1.0, "interval90_width": 1.0, "n": 375, "nrps": 0.17161399632774915, "score": 88.44295823839374, "top_ece": 0.029763419452187988 } }, "score": 86.60772459153094 }, "capability": 43.87421595931999, "composite_ci95": { "A": [ 4.213826902762912e-06, 0.0321843481372824 ], "B": [ 2.6407677436324784e-06, 0.02116413476940045 ], "C": [ 2.9262686824742456e-06, 0.022350241762001666 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: size-class proxy (deepinfra:Qwen/Qwen3-Reranker-0.6B); no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.020996016009852216 }, "display": "Mixedbread mxbai-rerank-base-v2", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 1.5821442040322362, "I_sealed": 0.6992704501858459, "base": 1.140707327109041, "excess": -1.6905138885918536, "gap": 0.8828737538463903, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 4.746432612096709, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": -5.081081081081076, "hard": -0.03145643284050508, "judge": 1.1206328279499007, "standard": 24.654667224780237 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": -11.030065754992213, "hard": -1.5426620524559675, "judge": 8.497895555670832, "standard": -21.144844232023672 } }, "sealed|choice": { "cc": 0.0, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": -7.75256817677922, "hard": -4.355203619909522, "judge": -1.4897579143388995, "standard": 0.23401896785319723 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "sealed|score": { "cc": 2.0978113505575378, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -4.830772427457819, "hard": 1.5240605072453395, "judge": 6.561942168694756, "standard": 0.013408698983785605 } } } }, "jevbench_score": 0.002273929358844764, "key": "mxbai-rerank-base-v2", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 85, "ranked": true, "ranks": { "A": 85, "B": 85, "C": 85, "capability": 62 }, "scores": { "A": 0.002273929358844764, "B": 0.0014520695133694007, "C": 0.001579117610308864 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.21285173026844859, "p50_s_raw": 0.031425865134224296, "p95_s_adjusted": 0.2589194315019995, "p95_s_raw": 0.05445971575099974 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 81.5709999496809, 89.28164941290872 ], "I": [ 0.13834674179555617, 2.8157994354473197 ], "capability": [ 41.27690031534251, 45.59940972334693 ], "composite": { "A": [ 4.213826902762912e-06, 0.0321843481372824 ], "B": [ 2.6407677436324784e-06, 0.02116413476940045 ], "C": [ 2.9262686824742456e-06, 0.022350241762001666 ] }, "seed": 69 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 86.60772459153094, "capability": 43.87421595931999, "intelligence": 1.140707327109041 }, "per_type": { "choice": { "accuracy": 0.2, "calibration": 86.28903899095539, "cc": -2.3085740268844592, "cc_by_type": { "choice": -2.3085740268844592 }, "cc_tiered": 2.3732163060483544, "cc_tiered_open": 4.746432612096709, "cc_tiered_sealed": 0.0, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 0.0 }, "noul": { "accuracy": 0.0, "calibration": 85.09117654524371, "cc": -100.0, "cc_by_type": { "noul": -100.0 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.3413333333333333, "calibration": 88.44295823839374, "cc": 0.3851131600845714, "cc_by_type": { "score": 0.3851131600845714 }, "cc_tiered": 1.0489056752787689, "cc_tiered_open": 0.0, "cc_tiered_sealed": 2.0978113505575378, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.3851131600845714 } }, "raw_on_A": { "calibration": 84.06228474550946, "capability": 42.68728856938789, "intelligence": 1.312292393266321 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.0017272292257436567, "Emphasis on Accuracy 60:20:20": 0.0009793418989392745, "Emphasis on Cost 20:20:60": 0.002776895162759242, "Emphasis on Speed 20:60:20": 0.002811535213497941, "Intelligence only": 0.0005937213753409678 }, "class": "reranker", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/mixedbread-ai/mxbai-rerank-base-v2", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "494M cross-encoder", "adapter_id": "images/mxbai-rerank-base-v2/spec.json:reranker_adapter.RerankerAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.0, "acc_among_decisive": null, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Convai Innovations", "axes": { "calibration": 33.655846666999985, "cost": 82.1551651086044, "intelligence": 0.2879184668737643, "speed": 77.82044954853706 }, "calibration": { "parts": { "choice": { "ece_hard": 0.44879250000000004, "ece_score": 10.241499999999991, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 10.241499999999991 }, "noul": { "abstention_rate": 0.28, "brier": 0.4157348964266666, "ece": 0.3921216000000001, "n": 375, "score": 21.575679999999974 }, "score": { "interval90_coverage": 0.896, "interval90_width": 0.7111111111111107, "n": 375, "nrps": 0.18850373331333325, "score": 69.150360001, "top_ece": 0.21424453333333335 } }, "score": 33.655846666999985 }, "capability": 16.971882566936873, "composite_ci95": { "A": [ 0.0, 0.016100086645554724 ], "B": [ 0.0, 0.010673895641907892 ], "C": [ 0.0, 0.01118061572607967 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: documented hosted-model estimate; no exact base-model floor applies. $0.01/M input, $0/M output; same-class hosted-encoder estimate; no exact-base market floor.", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.003933940886699508 }, "display": "Laya multilingual", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 0.3687761432977424, "I_sealed": 0.20706079044978623, "base": 0.2879184668737643, "excess": -2.4116722895902876, "gap": 0.16171535284795616, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 1.1063284298932272, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 2.702702702702708, "hard": -13.243158225857174, "judge": 9.03098220171391, "standard": 17.120133947258267 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -14.28571428571429, "hard": -30.434782608695656, "judge": -50.0, "standard": -44.44444444444444 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 13.373939393939372, "hard": 2.3077777777777753, "judge": -8.492881355932202, "standard": -37.68148148148147 } }, "sealed|choice": { "cc": 0.6211823713493587, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 2.889660778952063, "hard": -4.86425339366518, "judge": 5.959031657355701, "standard": 2.451040768567572 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 0.0, "hard": -30.097087378640776, "judge": -36.8421052631579, "standard": -29.5774647887324 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -0.387943262411361, "hard": -0.8122549019607739, "judge": -1.6932335329341575, "standard": -3.1987817258883444 } } } }, "jevbench_score": 3.7595668430455566e-05, "key": "laya-multilingual", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 86, "ranked": true, "ranks": { "A": 86, "B": 86, "C": 86, "capability": 90 }, "scores": { "A": 3.7595668430455566e-05, "B": 2.368099374667976e-05, "C": 2.6108103076705254e-05 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.8654558090260253, "p50_s_raw": 0.3577279045130126, "p95_s_adjusted": 1.908579030103283, "p95_s_raw": 0.8792895150516415 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 28.738769095461095, 38.12861857012684 ], "I": [ 0.0, 2.2482651295080083 ], "capability": [ 14.566920276697426, 19.746046606060705 ], "composite": { "A": [ 0.0, 0.016100086645554724 ], "B": [ 0.0, 0.010673895641907892 ], "C": [ 0.0, 0.01118061572607967 ] }, "seed": 57 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 33.655846666999985, "capability": 16.971882566936873, "intelligence": 0.2879184668737643 }, "per_type": { "choice": { "accuracy": 0.21733333333333332, "calibration": 10.241499999999991, "cc": -0.09188825630196516, "cc_by_type": { "choice": -0.09188825630196516 }, "cc_tiered": 0.863755400621293, "cc_tiered_open": 1.1063284298932272, "cc_tiered_sealed": 0.6211823713493587, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 0.0 }, "noul": { "accuracy": 0.35733333333333334, "calibration": 21.575679999999974, "cc": -28.53333333333333, "cc_by_type": { "noul": -28.53333333333333 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.296, "calibration": 69.150360001, "cc": -1.8657280701754209, "cc_by_type": { "score": -1.8657280701754209 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 32.7172206036182, "capability": 17.460072535779506, "intelligence": 2.202924467940814 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 2.843624303166903e-05, "Emphasis on Accuracy 60:20:20": 1.5873606797432713e-05, "Emphasis on Cost 20:20:60": 4.7066204506481234e-05, "Emphasis on Speed 20:60:20": 4.704809343432325e-05, "Intelligence only": 9.547035876864281e-06 }, "class": "system-one-open", "open": "yes", "licence": "apache-2.0, not gated", "repo": "https://huggingface.co/convaiinnovations/laya/tree/1c5edc17a7acd8701df6fc341c0d179f1c62c982/multilingual", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU, Sandy (AMD Ryzen 5 3600), 4 of 12 threads, nice -n 5, shared host, offline (HF_HUB_OFFLINE=1)", "underlying": "convaiinnovations/laya @ 1c5edc17a7acd8701df6fc341c0d179f1c62c982, multilingual/; mmBERT-base 322M, max_len 1024. Distinct from the published English root checkpoint. Fresh run from item 1 on 29 Sep; interrupted earlier runs were not stitched. In-process, offline, shared AMD Ryzen 5 3600 CPU, four threads. Measured latency includes shared-host conditions; it is not a controlled GPU comparison. The release owner independently adopted mmBERT as unlisted in the frozen pricing snapshots; the labelled same-class encoder estimate has zero cost delta against BAAI/bge-m3.", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.72, "acc_among_decisive": 0.4962962962962963, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "AltSlate Labs", "axes": { "calibration": 86.14595667337036, "cost": 96.9130801359906, "intelligence": 0.20278658812856143, "speed": 92.53493469610669 }, "calibration": { "parts": { "choice": { "ece_hard": 0.06436937500000006, "ece_score": 87.12612499999999, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 87.12612499999999 }, "noul": { "abstention_rate": 0.9893333333333333, "brier": 0.25290737730666696, "ece": 0.08273039999999988, "n": 375, "score": 83.45392000000002 }, "score": { "interval90_coverage": 0.9946666666666667, "interval90_width": 0.9942222222222223, "n": 375, "nrps": 0.17416936626444454, "score": 87.8578250201111, "top_ece": 0.0343370666666668 } }, "score": 86.14595667337036 }, "capability": 43.17437163074946, "composite_ci95": { "A": [ 0.0, 0.004427570625789243 ], "B": [ 0.0, 0.0028324322673094096 ], "C": [ 0.0, 0.003074701823464752 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0012673522167487683 }, "display": "Certo v1 (AltSlate Labs)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 0.12963682017919345, "I_sealed": 0.27593635607792943, "base": 0.20278658812856143, "excess": -2.71968717833698, "gap": -0.14629953589873598, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 0.0, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 10.486486486486491, "hard": 1.855929537590448, "judge": -18.655240606460122, "standard": 2.051067392214312 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -100.0, "hard": -91.30434782608697, "judge": -100.0, "standard": -100.0 } }, "open|score": { "cc": 0.3889104605375804, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 5.111515151515156, "hard": -0.6524999999999892, "judge": 1.8106779661016925, "standard": -2.0222222222222364 } }, "sealed|choice": { "cc": 0.6168084160817238, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": -6.42228955731281, "hard": -4.86425339366518, "judge": -0.5586592178770757, "standard": 16.861682473210998 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "sealed|score": { "cc": 0.21100065215206465, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -0.7955319148936102, "hard": 1.9993137254901705, "judge": -1.0858682634730998, "standard": -0.9170558375635629 } } } }, "jevbench_score": 1.3254531668229452e-05, "key": "certo", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 87, "ranked": true, "ranks": { "A": 87, "B": 87, "C": 87, "capability": 64 }, "scores": { "A": 1.3254531668229452e-05, "B": 8.311485824043969e-06, "C": 9.204535880714898e-06 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.23407380385324358, "p50_s_raw": 0.042036901926621795, "p95_s_adjusted": 0.2383162835612893, "p95_s_raw": 0.04415814178064466 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 80.84167298672317, 89.02661690205646 ], "I": [ 0.0, 1.4261709005081855 ], "capability": [ 40.54746106861931, 44.91215541214051 ], "composite": { "A": [ 0.0, 0.004427570625789243 ], "B": [ 0.0, 0.0028324322673094096 ], "C": [ 0.0, 0.003074701823464752 ] }, "seed": 19 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 86.14595667337036, "capability": 43.17437163074946, "intelligence": 0.20278658812856143 }, "per_type": { "choice": { "accuracy": 0.21733333333333332, "calibration": 87.12612499999999, "cc": -0.09188825630196516, "cc_by_type": { "choice": -0.09188825630196516 }, "cc_tiered": 0.3084042080408619, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.6168084160817238, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 0.0 }, "noul": { "accuracy": 0.0026666666666666666, "calibration": 83.45392000000002, "cc": -99.46666666666667, "cc_by_type": { "noul": -99.46666666666667 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.27466666666666667, "calibration": 87.8578250201111, "cc": 0.4315701754385892, "cc_by_type": { "score": 0.4315701754385892 }, "cc_tiered": 0.2999555563448225, "cc_tiered_open": 0.3889104605375804, "cc_tiered_sealed": 0.21100065215206465, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.4315701754385892 } }, "raw_on_A": { "calibration": 84.87407902121794, "capability": 42.46944871565377, "intelligence": 0.06481841008959673 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 9.964199692814982e-06, "Emphasis on Accuracy 60:20:20": 5.551453514139297e-06, "Emphasis on Cost 20:20:60": 1.653808420915537e-05, "Emphasis on Speed 20:60:20": 1.6534837758955257e-05, "Intelligence only": 3.3356285030110653e-06 }, "class": "jev-rebuild", "open": "yes", "licence": "MIT", "repo": "https://huggingface.co/altslate/certo-decision-model", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "ModernBERT-large with a per-option query/scoring head, ~400M parameters", "adapter_id": "images/certo/spec.json:certo_v15.CertoLocalV15Adapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.010666666666666666, "acc_among_decisive": 0.25, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Contrastive-LM (Kwok, Kang, Suresh, Saad-Falcon, Pavone, Ré, Mirhoseini)", "axes": { "calibration": 40.041900621338065, "cost": 50.501662240594015, "intelligence": 0.13075619159747265, "speed": 92.63548383949004 }, "calibration": { "parts": { "choice": { "ece_hard": 0.3993510766836583, "ece_score": 20.129784663268335, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 20.129784663268335 }, "noul": { "abstention_rate": 0.4906666666666667, "brier": 0.26050031793956074, "ece": 0.18460455903608142, "n": 375, "score": 63.079088192783715 }, "score": { "interval90_coverage": 0.688, "interval90_width": 0.5184444444444443, "n": 375, "nrps": 0.2846437031480146, "score": 36.91682900796216, "top_ece": 0.48850985834637106 } }, "score": 40.041900621338065 }, "capability": 20.086328406467768, "composite_ci95": { "A": [ 0.0, 0.010708585645302497 ], "B": [ 0.0, 0.007040756824095157 ], "C": [ 0.0, 0.007436517809237845 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.044662660714285714 }, "display": "CLM-8B (Contrastive-LM, clm-latest)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 0.2615123831949453, "I_sealed": 0.0, "base": 0.13075619159747265, "excess": -2.3118752592432985, "gap": 0.2615123831949453, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 0.7845371495848359, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": -16.75675675675675, "hard": -7.5810003145643154, "judge": 16.941331575477918, "standard": 2.051067392214312 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -28.57142857142857, "hard": -56.52173913043479, "judge": -19.999999999999996, "standard": -44.44444444444444 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": -21.94096032005719, "hard": -1.9559175706170606, "judge": -11.452540725941729, "standard": -95.63691251131768 } }, "sealed|choice": { "cc": 0.0, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": -6.42228955731281, "hard": -4.355203619909522, "judge": 0.3724394785847514, "standard": 7.993595270353507 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -31.999999999999996, "hard": -33.98058252427184, "judge": -31.57894736842105, "standard": -9.859154929577462 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -10.75981978481908, "hard": -29.341375883900135, "judge": -28.232663703069093, "standard": -55.86396111215961 } } } }, "jevbench_score": 3.5510969933114796e-06, "key": "clm-8b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 88, "ranked": true, "ranks": { "A": 88, "B": 88, "C": 88, "capability": 88 }, "scores": { "A": 3.5510969933114796e-06, "B": 2.2274698008760027e-06, "C": 2.466039578688528e-06 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.19238147847354412, "p50_s_raw": 0.02119073923677206, "p95_s_adjusted": 0.2833272576332092, "p95_s_raw": 0.0666636288166046 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 35.39195795358793, 44.454662318208044 ], "I": [ 0.0, 1.9510432807841849 ], "capability": [ 17.69968780119306, 22.526421025586746 ], "composite": { "A": [ 0.0, 0.010708585645302497 ], "B": [ 0.0, 0.007040756824095157 ], "C": [ 0.0, 0.007436517809237845 ] }, "seed": 22 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 40.041900621338065, "capability": 20.086328406467768, "intelligence": 0.13075619159747265 }, "per_type": { "choice": { "accuracy": 0.20533333333333334, "calibration": 20.129784663268335, "cc": -1.6265168667052297, "cc_by_type": { "choice": -1.6265168667052297 }, "cc_tiered": 0.39226857479241795, "cc_tiered_open": 0.7845371495848359, "cc_tiered_sealed": 0.0, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 0.0 }, "noul": { "accuracy": 0.352, "calibration": 63.079088192783715, "cc": -29.600000000000005, "cc_by_type": { "noul": -29.600000000000005 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.26666666666666666, "calibration": 36.91682900796216, "cc": -29.29970040054666, "cc_by_type": { "score": -29.29970040054666 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 38.495636537708954, "capability": 19.549069061819143, "intelligence": 0.6025015859293292 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 2.67198512793619e-06, "Emphasis on Accuracy 60:20:20": 1.488390058441499e-06, "Emphasis on Cost 20:20:60": 4.430457758214625e-06, "Emphasis on Speed 20:60:20": 4.4408219452037955e-06, "Intelligence only": 8.942249433748766e-07 }, "class": "system-one-open", "open": "yes", "licence": "Apache-2.0 (code and CLM-v0.1-8B head); Qwen3-8B encoder Apache-2.0", "repo": "https://github.com/Contrastive-LM/CLM", "gpu": "RTXPRO6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container", "underlying": null, "adapter_id": "images/clm-8b/spec.json:clm_adapter.CLMAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5093333333333333, "acc_among_decisive": 0.6910994764397905, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "BAAI", "axes": { "calibration": 88.10046932372114, "cost": 59.325605115070715, "intelligence": 0.06512222845080358, "speed": 93.86976236118387 }, "calibration": { "parts": { "choice": { "ece_hard": 0.027473000546109336, "ece_score": 94.50539989077814, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 94.50539989077814 }, "noul": { "abstention_rate": 1.0, "brier": 0.25096294008809744, "ece": 0.0684901503152521, "n": 375, "score": 86.30196993694959 }, "score": { "interval90_coverage": 1.0, "interval90_width": 1.0, "n": 375, "nrps": 0.1745560185107233, "score": 83.49403814343572, "top_ece": 0.07778160931028115 } }, "score": 88.10046932372114 }, "capability": 44.08279577608597, "composite_ci95": { "A": [ 0.0, 0.008573668302290527 ], "B": [ 0.0, 0.005538697219146524 ], "C": [ 0.0, 0.005953936321035087 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: base-model market reference (deepinfra:BAAI/bge-m3)", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.022688885467980296 }, "display": "BAAI bge-reranker-v2-m3", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 0.018692443590466585, "I_sealed": 0.11155201331114058, "base": 0.06512222845080358, "excess": -2.666247212158918, "gap": -0.092859569720674, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 0.05607733077139976, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 10.486486486486491, "hard": 7.518087448883308, "judge": -14.700065919578117, "standard": 2.051067392214312 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 5.306919494288776, "hard": -1.440659315203363, "judge": -2.980317227364382, "standard": -2.397579483854617 } }, "sealed|choice": { "cc": 0.33465603993342175, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": -5.092010937846398, "hard": -0.791855203619931, "judge": 2.234636871508399, "standard": 2.451040768567572 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 2.319622122935927, "hard": -0.38388489349363564, "judge": 0.07005074041354087, "standard": -0.8557298803223867 } } } }, "jevbench_score": 4.4076803151681546e-07, "key": "bge-reranker-v2-m3", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 89, "ranked": true, "ranks": { "A": 89, "B": 89, "C": 89, "capability": 60 }, "scores": { "A": 4.4076803151681546e-07, "B": 2.758281497029271e-07, "C": 3.060889107755663e-07 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.1841465687379241, "p50_s_raw": 0.01707328436896205, "p95_s_adjusted": 0.2227717584464699, "p95_s_raw": 0.03638587922323495 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 83.14061191725726, 90.57274307882632 ], "I": [ 0.0, 1.7895886627489592 ], "capability": [ 41.80444040346114, 45.59879463409008 ], "composite": { "A": [ 0.0, 0.008573668302290527 ], "B": [ 0.0, 0.005538697219146524 ], "C": [ 0.0, 0.005953936321035087 ] }, "seed": 18 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 88.10046932372114, "capability": 44.08279577608597, "intelligence": 0.06512222845080358 }, "per_type": { "choice": { "accuracy": 0.22133333333333333, "calibration": 94.50539989077814, "cc": 0.4196546138324576, "cc_by_type": { "choice": 0.4196546138324576 }, "cc_tiered": 0.19536668535241075, "cc_tiered_open": 0.05607733077139976, "cc_tiered_sealed": 0.33465603993342175, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 0.4196546138324576 }, "noul": { "accuracy": 0.0, "calibration": 86.30196993694959, "cc": -100.0, "cc_by_type": { "noul": -100.0 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.2826666666666667, "calibration": 83.49403814343572, "cc": -0.18199548701309798, "cc_by_type": { "score": -0.18199548701309798 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 85.43472005366168, "capability": 43.01708899298245, "intelligence": 0.5994579323032229 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 3.308199423158738e-07, "Emphasis on Accuracy 60:20:20": 1.8400822603920523e-07, "Emphasis on Cost 20:20:60": 5.501608977147286e-07, "Emphasis on Speed 20:60:20": 5.506039727293195e-07, "Intelligence only": 1.1047086427994778e-07 }, "class": "reranker", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/BAAI/bge-reranker-v2-m3", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "568M multilingual cross-encoder", "adapter_id": "images/bge-reranker-v2-m3/spec.json:reranker_adapter.RerankerAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.0, "acc_among_decisive": null, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Alibaba-NLP", "axes": { "calibration": 83.4556233651092, "cost": 69.26965431492843, "intelligence": 0.0, "speed": 93.76120791185616 }, "calibration": { "parts": { "choice": { "ece_hard": 0.0790764439307023, "ece_score": 84.18471121385954, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 84.18471121385954 }, "noul": { "abstention_rate": 1.0, "brier": 0.25231019651494707, "ece": 0.08962639509020176, "n": 375, "score": 82.07472098195964 }, "score": { "interval90_coverage": 1.0, "interval90_width": 1.0, "n": 375, "nrps": 0.17667560258952839, "score": 84.10743789950841, "top_ece": 0.07058781971015167 } }, "score": 83.4556233651092 }, "capability": 41.7278116825546, "composite_ci95": { "A": [ 0.0, 0.003662687483805933 ], "B": [ 0.0, 0.0023440984279851505 ], "C": [ 0.0, 0.002543532974865231 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: size-class proxy (deepinfra:thenlper/gte-base); no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.010576570197044334 }, "display": "Alibaba GTE Reranker ModernBERT-base", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 0.0, "I_sealed": 0.0, "base": 0.0, "excess": -2.573387642438244, "gap": 0.0, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 0.0, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 6.5945945945945965, "hard": -3.8062283737024116, "judge": -6.789716545814108, "standard": 9.585600669736289 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": -1.920002214440264, "hard": -1.3526386292960213, "judge": 1.2639659476201492, "standard": 0.3612908498087819 } }, "sealed|choice": { "cc": 0.0, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": -5.092010937846398, "hard": -6.391402714932147, "judge": -5.214152700186197, "standard": -7.525557334647108 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -1.317182445372178, "hard": 0.03511410774166279, "judge": -0.2802076730282099, "standard": 0.30690766431314964 } } } }, "jevbench_score": 0.0, "key": "gte-reranker-modernbert-base", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 90, "ranked": true, "ranks": { "A": 90, "B": 90, "C": 90, "capability": 67 }, "scores": { "A": 0.0, "B": 0.0, "C": 0.0 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.19835810186341404, "p50_s_raw": 0.024179050931707025, "p95_s_adjusted": 0.2120455999393016, "p95_s_raw": 0.031022799969650804 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.59536793549519, 87.51314141306226 ], "I": [ 0.0, 1.339161261347401 ], "capability": [ 39.372795544944715, 43.89015023558699 ], "composite": { "A": [ 0.0, 0.003662687483805933 ], "B": [ 0.0, 0.0023440984279851505 ], "C": [ 0.0, 0.002543532974865231 ] }, "seed": 39 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 83.4556233651092, "capability": 41.7278116825546, "intelligence": 0.0 }, "per_type": { "choice": { "accuracy": 0.17866666666666667, "calibration": 84.18471121385954, "cc": -5.036802667601379, "cc_by_type": { "choice": -5.036802667601379 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 0.0 }, "noul": { "accuracy": 0.0, "calibration": 82.07472098195964, "cc": -100.0, "cc_by_type": { "noul": -100.0 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.304, "calibration": 84.10743789950841, "cc": -0.24811073820258933, "cc_by_type": { "score": -0.24811073820258933 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 88.93979403901187, "capability": 44.469897019505936, "intelligence": 0.0 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.0, "Emphasis on Accuracy 60:20:20": 0.0, "Emphasis on Cost 20:20:60": 0.0, "Emphasis on Speed 20:60:20": 0.0, "Intelligence only": 0.0 }, "class": "reranker", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/Alibaba-NLP/gte-reranker-modernbert-base", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "149M ModernBERT cross-encoder", "adapter_id": "images/gte-reranker-modernbert-base/spec.json:reranker_adapter.RerankerAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.0, "acc_among_decisive": null, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Bluusun", "axes": { "calibration": 51.61277239297377, "cost": 89.27300805786774, "intelligence": 0.0, "speed": 73.47211485993975 }, "calibration": { "parts": { "choice": { "ece_hard": 0.2204942558257705, "ece_score": 55.90114883484589, "mean_tvd": null, "n": 379, "n_ece_hard": 100, "n_tvd": 0, "score": 55.90114883484589 }, "noul": { "abstention_rate": 0.07773851590106007, "brier": 0.3825718867287919, "ece": 0.37640864622953524, "n": 283, "score": 24.718270754092952 }, "score": { "interval90_coverage": 0.9471544715447154, "interval90_width": 0.8113143631436313, "n": 246, "nrps": 0.18235540291655192, "score": 74.21889758998248, "top_ece": 0.1666333226418992 } }, "score": 51.61277239297377 }, "capability": 25.806386196486883, "composite_ci95": { "A": [ 0.0, 9.853474730694675e-09 ], "B": [ 0.0, 6.161443172813211e-09 ], "C": [ 0.0, 6.8426907852046335e-09 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0022780665024630543 }, "display": "Mirror", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 0.0, "I_sealed": 0.0, "base": 0.0, "excess": -2.573387642438244, "gap": 0.0, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 0.0, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 6.5945945945945965, "hard": -7.5810003145643154, "judge": -14.700065919578117, "standard": -5.4834658853076625 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 0.0, "hard": -73.91304347826086, "judge": -19.999999999999996, "standard": 22.222222222222232 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 14.02621656810803, "hard": -86.48465684738163, "judge": -131.29938184642268, "standard": -61.876452149889175 } }, "sealed|choice": { "cc": 0.0, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 14.86216835414975, "hard": -16.063348416289617, "judge": -12.662942271880798, "standard": -6.417046434289924 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 12.00000000000001, "hard": -33.98058252427184, "judge": -15.789473684210531, "standard": -7.042253521126762 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -1.6525187057042423, "hard": -113.52957658615944, "judge": -73.84737475308705, "standard": -29.673128809498152 } } } }, "jevbench_score": 0.0, "key": "mirror", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 91, "ranked": true, "ranks": { "A": 91, "B": 91, "C": 91, "capability": 87 }, "scores": { "A": 0.0, "B": 0.0, "C": 0.0 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 1.4516588259488343, "p50_s_raw": 0.6508294129744172, "p95_s_adjusted": 3.096876988979056, "p95_s_raw": 1.473438494489528 }, "status": { "answered_ok": 908, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 42.56771416804497, 56.74054715693854 ], "I": [ 0.0, 0.01760273906633102 ], "capability": [ 21.283857084022486, 28.426215016960548 ], "composite": { "A": [ 0.0, 9.853474730694675e-09 ], "B": [ 0.0, 6.161443172813211e-09 ], "C": [ 0.0, 6.8426907852046335e-09 ] }, "seed": 68 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 51.61277239297377, "capability": 25.806386196486883, "intelligence": 0.0 }, "per_type": { "choice": { "accuracy": 0.156, "calibration": 55.90114883484589, "cc": -7.935545598363106, "cc_by_type": { "choice": -7.935545598363106 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 0.0 }, "noul": { "accuracy": 0.416, "calibration": 24.718270754092952, "cc": -16.800000000000004, "cc_by_type": { "noul": -16.800000000000004 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.18666666666666668, "calibration": 74.21889758998248, "cc": -72.82982094179775, "cc_by_type": { "score": -72.82982094179775 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 48.60190522372855, "capability": 24.300952611864275, "intelligence": 0.0 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.4946666666666667, "n": 750 }, "noul": { "invalid_rate": 0.2453333333333333, "n": 375 }, "score": { "invalid_rate": 0.344, "n": 375 } }, "views": { "Balanced 33:33:33": 0.0, "Emphasis on Accuracy 60:20:20": 0.0, "Emphasis on Cost 20:20:60": 0.0, "Emphasis on Speed 20:60:20": 0.0, "Intelligence only": 0.0 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 wrapper and Mirror release; upstream DeBERTa/model assets retain their own terms", "repo": "https://github.com/Bluusun/Decision-API", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "underlying": "Mirror DeBERTa-v3-large 436M classification-span scorer", "adapter_id": "images/mirror/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.9222614840989399, "acc_among_decisive": 0.5977011494252874, "valid": 283 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "JoshuaSP", "axes": { "calibration": 0.0, "cost": 49.295219536461964, "intelligence": 61.19221807027989, "speed": 85.53332924416175 }, "calibration": { "parts": {}, "score": 0.0 }, "capability": 30.596109035139946, "composite_ci95": { "A": [ 0.0, 0.0 ], "B": [ 0.0, 0.0 ], "C": [ 0.0, 0.0 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.04899585591133005 }, "display": "Open Jev JSON Canvas (JoshuaSP)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 61.36340599482574, "I_sealed": 61.02103014573405, "base": 61.19221807027989, "excess": -2.2310117933465556, "gap": 0.34237584909168817, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 60.21617418117772, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 100.00000000000001, "hard": 49.040578798364265, "judge": 60.44825313117996, "standard": 62.32733361239012 } }, "open|noul": { "cc": 72.15458937198068, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 82.6086956521739, "judge": 60.00000000000001, "standard": 55.55555555555556 } }, "open|score": { "cc": 51.71945443131884, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 54.54545454545454, "hard": 38.88888888888889, "judge": 67.79661016949152, "standard": 51.85185185185186 } }, "sealed|choice": { "cc": 68.74482965663773, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 77.38526346907102, "hard": 60.80316742081447, "judge": 71.13594040968344, "standard": 76.7212710924991 } }, "sealed|noul": { "cc": 64.47674149136002, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 84.00000000000001, "hard": 45.63106796116505, "judge": 76.3157894736842, "standard": 74.64788732394365 } }, "sealed|score": { "cc": 49.84151928920441, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 70.92198581560282, "hard": 46.81372549019608, "judge": 50.29940119760479, "standard": 44.6700507614213 } } } }, "jevbench_score": 0.0, "key": "open-jev-json-canvas-joshuasp", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 92, "ranked": true, "ranks": { "A": 92, "B": 92, "C": 92, "capability": 83 }, "scores": { "A": 0.0, "B": 0.0, "C": 0.0 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.44667022498324516, "p50_s_raw": 0.14833511249162257, "p95_s_adjusted": 0.6261524326633662, "p95_s_raw": 0.23807621633168308 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "label", "noul": "label", "score": "label" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 0.0, 0.0 ], "I": [ 54.979058028212734, 65.63316060204029 ], "capability": [ 27.489529014106367, 32.816580301020146 ], "composite": { "A": [ 0.0, 0.0 ], "B": [ 0.0, 0.0 ], "C": [ 0.0, 0.0 ] }, "seed": 76 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 0.0, "capability": 30.596109035139946, "intelligence": 61.19221807027989 }, "per_type": { "choice": { "accuracy": 0.7493333333333333, "calibration": null, "cc": 67.9433134715762, "cc_by_type": { "choice": 67.9433134715762 }, "cc_tiered": 64.48050191890772, "cc_tiered_open": 60.21617418117772, "cc_tiered_sealed": 68.74482965663773, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 67.9433134715762 }, "noul": { "accuracy": 0.84, "calibration": null, "cc": 68.0, "cc_by_type": { "noul": 68.0 }, "cc_tiered": 68.31566543167034, "cc_tiered_open": 72.15458937198068, "cc_tiered_sealed": 64.47674149136002, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 68.0 }, "score": { "accuracy": 0.624, "calibration": null, "cc": 50.6140350877193, "cc_by_type": { "score": 50.6140350877193 }, "cc_tiered": 50.780486860261625, "cc_tiered_open": 51.71945443131884, "cc_tiered_sealed": 49.84151928920441, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 50.6140350877193 } }, "raw_on_A": { "calibration": 0.0, "capability": 31.028102732437542, "intelligence": 62.056205464875084 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 60.34909628466887, "Emphasis on Accuracy 60:20:20": 59.99813997005745, "Emphasis on Cost 20:20:60": 54.674045042588084, "Emphasis on Speed 20:60:20": 67.78111653907041, "Intelligence only": 59.47929292694184 }, "class": "jev-rebuild", "open": "yes", "licence": "MIT code; Apache-2.0 DiffusionGemma weights", "repo": "https://github.com/JoshuaSP/open-jev", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "google/diffusiongemma-26B-A4B-it BF16, one-step JSON canvas", "adapter_id": "images/open-jev-json-canvas-joshuasp/spec.json:joshuasp_adapter.JoshuaSPCanvasAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 1.0, "acc_among_decisive": 0.8490566037735849, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": true, "api_subset": true, "author": "Wity (alphanimble)", "axes": { "calibration": 90.96578585343975, "cost": 58.368182152360596, "intelligence": 70.45905034996007, "speed": 71.20296182763504 }, "calibration": { "parts": { "choice": { "ece_hard": 0.07112499977806429, "ece_score": 85.77500004438714, "mean_tvd": null, "n": 300, "n_ece_hard": 128, "n_tvd": 0, "score": 85.77500004438714 }, "noul": { "abstention_rate": 0.10666666666666669, "brier": 0.08500644420882862, "ece": 0.05074052247778312, "n": 150, "score": 89.85189550444338 }, "score": { "interval90_coverage": 0.9733333333333334, "interval90_width": 0.205, "n": 150, "nrps": 0.05354926203758897, "score": 92.57898435089088, "top_ece": 0.04743552547229674 } }, "score": 89.4019599665738 }, "capability": 80.71241810169991, "composite_ci95": { "A": [ 68.00237639540825, 73.17586873338328 ], "B": [ 66.86715175820622, 73.95159144532194 ], "C": [ 67.9906235937158, 73.17586873338328 ] }, "cost": { "basis": "Wity's stated API tariff: USD 0.042 per 1M input tokens, output free; v1.5 cost per 1,000 decisions carried from the v1.5.8 build (measured 944,200 input tokens over 1,624 decisions, build f65ce826a455). API models with a known base are ranked at their own API price (1 Oct 2026 rule).", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.02441896551724138 }, "display": "wity-1 (Wity, reasoning always)", "endpoint_kind": "api", "full_coverage": true, "intelligence": { "I_open": 67.76360137882253, "I_sealed": 73.73193266458848, "base": 70.7477670217055, "excess": -9.793263332885296, "gap": -5.968331285765956, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 82.46797884719487, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 73.57659641396666, "judge": 88.13447593935398, "standard": 84.93093344495604 } }, "open|noul": { "cc": 58.719806763285035, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 56.52173913043479, "judge": 50.0, "standard": 55.55555555555556 } }, "open|score": { "cc": 62.10301852598769, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 75.97327196189137, "hard": 60.98101749360574, "judge": 66.87830350526892, "standard": 50.24896640387793 } }, "sealed|choice": { "cc": 82.46814294181557, "n": { "easy": 21, "hard": 60, "judge": 34, "standard": 35 }, "tiers": { "easy": 93.88794567062817, "hard": 74.50424929178472, "judge": 84.87077117041395, "standard": 89.08208653457339 } }, "sealed|noul": { "cc": 71.50058769753167, "n": { "easy": 13, "hard": 31, "judge": 12, "standard": 19 }, "tiers": { "easy": 84.61538461538463, "hard": 48.38709677419355, "judge": 100.0, "standard": 68.42105263157893 } }, "sealed|score": { "cc": 67.22706735441821, "n": { "easy": 12, "hard": 31, "judge": 12, "standard": 20 }, "tiers": { "easy": 93.63618279843891, "hard": 61.26496713065269, "judge": 56.019982030255576, "standard": 82.75733806618283 } } } }, "jevbench_score": 70.96883700634847, "key": "wity-1-always", "listing": "listed", "not_ranked_because": "registry ranked=false or no composite", "planned_weight_denominator": 1.0, "rank": null, "ranked": false, "scores": { "A": 70.96883700634847, "B": 70.86629038091284, "C": 70.96883700634847 }, "speed": { "adjustment": "none (API)", "n": 124, "p50_s_adjusted": 1.6885605081915855, "p50_s_raw": 1.6885605081915855, "p95_s_adjusted": 4.489388496428728, "p95_s_raw": 4.489388496428728 }, "status": { "answered_ok": 600, "missing": 0, "rows": 600, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": true, "api_subset_tag": "A2", "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 84.03611172903366, 98.06554947931198 ], "I": [ 62.12520278512642, 77.83709243724311 ], "capability": [ 74.25018288527112, 86.23115265506914 ], "composite": { "A": [ 68.00237639540825, 73.17586873338328 ], "B": [ 66.86715175820622, 73.95159144532194 ], "C": [ 67.9906235937158, 73.17586873338328 ] }, "seed": 111 }, "ci95_basis": "bootstrap over the API system's items (n = A sealed items) AND over the equating offset (pool systems resampled with replacement, items resampled; B_offset = 200)", "ci95_n": 300, "complete": true, "equated": { "calibration": 90.96578585343975, "capability": 80.71241810169991, "intelligence": 70.45905034996007 }, "equating_flag": null, "equating_offset": { "C": 1.5638258868659562, "I": -0.2887166717454388 }, "g_med_used": 3.824932047119341, "lane": "api", "metric_basis": "A2 u P (sealed side = supplementary API subset A2, n=300)", "n_items": 600, "n_public": 300, "n_sealed_side": 300, "offset_ci95": { "C": [ -1.2071304620902201, 9.442982800888801 ], "I": [ -5.855912654795828, 4.37048407464939 ] }, "on_S": null, "per_type": { "choice": { "accuracy": 0.8666666666666667, "calibration": 85.77500004438714, "cc": 82.94808639636226, "cc_by_type": { "choice": 82.94808639636226 }, "cc_tiered": 82.46806089450521, "cc_tiered_open": 82.46797884719487, "cc_tiered_sealed": 82.46814294181557, "low_n": false, "n": 300, "n_by_type": { "choice": 300 }, "score": 82.94808639636226 }, "noul": { "accuracy": 0.8266666666666667, "calibration": 89.85189550444338, "cc": 65.33333333333333, "cc_by_type": { "noul": 65.33333333333333 }, "cc_tiered": 65.11019723040835, "cc_tiered_open": 58.719806763285035, "cc_tiered_sealed": 71.50058769753167, "low_n": false, "n": 150, "n_by_type": { "noul": 150 }, "score": 65.33333333333333 }, "score": { "accuracy": 0.8, "calibration": 92.57898435089088, "cc": 67.78602533660896, "cc_by_type": { "score": 67.78602533660896 }, "cc_tiered": 64.66504294020295, "cc_tiered_open": 62.10301852598769, "cc_tiered_sealed": 67.22706735441821, "low_n": false, "n": 150, "n_by_type": { "score": 150 }, "score": 67.78602533660896 } }, "raw_on_A": { "calibration": 89.4019599665738, "capability": 80.07486349413965, "intelligence": 70.7477670217055 }, "set": "A2+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 300 }, "noul": { "invalid_rate": 0.0, "n": 150 }, "score": { "invalid_rate": 0.0, "n": 150 } }, "views": { "Balanced 33:33:33": 66.12353898481751, "Emphasis on Accuracy 60:20:20": 67.79210208696036, "Emphasis on Cost 20:20:60": 62.786568563342435, "Emphasis on Speed 20:60:20": 68.06578922688111, "Intelligence only": 70.45905034996007 }, "model_pin": null, "last_measured_on": null, "measurement_date_status": "unknown", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.8933333333333333, "acc_among_decisive": 0.9253731343283582, "valid": 150 }, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "author submission" }, "class": "decision-api", "open": "no", "licence": "proprietary hosted API", "repo": "https://wity.alphanimble.com/", "endpoint_condition": "operator's hosted production API, measured as is; API default is off, our request sets reasoning=always; labelled configuration variant.", "underlying": "Qwen3.6-35B-A3B (operator self-reported; not independently verified)", "adapter_id": "run/wity-1-always.spec.json:wity_reasoning_adapter.WityReasoningAdapter", "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "alt": { "label": "Qwen3.6-35B-A3B base-model reference price", "note": "Operator self-reported base, not independently verified; frozen Qwen/Qwen3.6-35B-A3B reference USD 0.15/M input and 1.00/M output, applied to the same measured tokens.", "usd_per_1000": 0.08721059113300492, "axes": { "calibration": 91.0421772337923, "cost": 41.78292309262718, "intelligence": 79.49960112075699, "speed": 74.36141604169042 } } }, { "addendum": null, "api_flag": true, "api_subset": true, "author": "Wity (alphanimble)", "axes": { "calibration": 74.90911730698414, "cost": 58.368182152360596, "intelligence": 40.91725354389175, "speed": 89.55912090143326 }, "calibration": { "parts": { "choice": { "ece_hard": 0.16490082004259887, "ece_score": 67.01983599148022, "mean_tvd": null, "n": 300, "n_ece_hard": 128, "n_tvd": 0, "score": 67.01983599148022 }, "noul": { "abstention_rate": 0.26, "brier": 0.15876844741124826, "ece": 0.14438367821577944, "n": 150, "score": 71.1232643568441 }, "score": { "interval90_coverage": 0.9266666666666666, "interval90_width": 0.48888888888888904, "n": 150, "nrps": 0.10193379084677297, "score": 81.89277391203026, "top_ece": 0.13010536545631077 } }, "score": 73.34529142011819 }, "capability": 57.913185425437945, "composite_ci95": { "A": [ 20.344473547585384, 60.79316245974321 ], "B": [ 17.756438192517088, 57.05281493247512 ], "C": [ 14.128106630267629, 42.21747393037723 ] }, "cost": { "basis": "Wity's stated API tariff: USD 0.042 per 1M input tokens, output free; v1.5 cost per 1,000 decisions carried from the v1.5.8 build (measured 944,200 input tokens over 1,624 decisions, build f65ce826a455). API models with a known base are ranked at their own API price (1 Oct 2026 rule).", "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.02441896551724138 }, "display": "wity-1 (Wity, reasoning off)", "endpoint_kind": "api", "full_coverage": true, "intelligence": { "I_open": 41.233518722452544, "I_sealed": 41.17842170882184, "base": 41.205970215637194, "excess": -3.7698350334886337, "gap": 0.05509701363070718, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 60.211735029468755, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 80.54054054054055, "hard": 50.92796476879523, "judge": 64.40342781806196, "standard": 62.32733361239012 } }, "open|noul": { "cc": 29.155969634230512, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 21.739130434782616, "judge": 10.000000000000009, "standard": 44.44444444444444 } }, "open|score": { "cc": 34.332851503658375, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 66.55807900089735, "hard": 31.747271308753078, "judge": 45.35548485808892, "standard": 6.857448113203645 } }, "sealed|choice": { "cc": 56.84865743652719, "n": { "easy": 21, "hard": 60, "judge": 34, "standard": 35 }, "tiers": { "easy": 87.77589134125635, "hard": 40.50991501416432, "judge": 73.5238495482244, "standard": 49.04973716134249 } }, "sealed|noul": { "cc": 22.111793130468854, "n": { "easy": 13, "hard": 31, "judge": 12, "standard": 19 }, "tiers": { "easy": 84.61538461538463, "hard": -29.032258064516125, "judge": 66.66666666666667, "standard": 26.315789473684205 } }, "sealed|score": { "cc": 44.57481455946947, "n": { "easy": 12, "hard": 31, "judge": 12, "standard": 20 }, "tiers": { "easy": 69.00023322090838, "hard": 37.40887238609477, "judge": 41.43728585626254, "standard": 51.400282630309825 } } } }, "jevbench_score": 40.53344853152938, "key": "wity-1-off", "listing": "listed", "not_ranked_because": "registry ranked=false or no composite", "planned_weight_denominator": 1.0, "rank": null, "ranked": false, "scores": { "A": 40.53344853152938, "B": 36.98830680062365, "C": 28.148228146895406 }, "speed": { "adjustment": "none (API)", "n": 124, "p50_s_adjusted": 0.28173011541366577, "p50_s_raw": 0.28173011541366577, "p95_s_adjusted": 0.39287521950900556, "p95_s_raw": 0.39287521950900556 }, "status": { "answered_ok": 600, "missing": 0, "rows": 600, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": true, "api_subset_tag": "A2", "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 66.12529271169734, 82.54593776435362 ], "I": [ 30.870972564797494, 48.610177293893685 ], "capability": [ 50.16674128124488, 64.09951024329372 ], "composite": { "A": [ 20.344473547585384, 60.79316245974321 ], "B": [ 17.756438192517088, 57.05281493247512 ], "C": [ 14.128106630267629, 42.21747393037723 ] }, "seed": 114 }, "ci95_basis": "bootstrap over the API system's items (n = A sealed items) AND over the equating offset (pool systems resampled with replacement, items resampled; B_offset = 200)", "ci95_n": 300, "complete": true, "equated": { "calibration": 74.90911730698414, "capability": 57.913185425437945, "intelligence": 40.91725354389175 }, "equating_flag": null, "equating_offset": { "C": 1.5638258868659562, "I": -0.2887166717454388 }, "g_med_used": 3.824932047119341, "lane": "api", "metric_basis": "A2 u P (sealed side = supplementary API subset A2, n=300)", "n_items": 600, "n_public": 300, "n_sealed_side": 300, "offset_ci95": { "C": [ -1.2071304620902201, 9.442982800888801 ], "I": [ -5.855912654795828, 4.37048407464939 ] }, "on_S": null, "per_type": { "choice": { "accuracy": 0.68, "calibration": 67.01983599148022, "cc": 59.075407351269405, "cc_by_type": { "choice": 59.075407351269405 }, "cc_tiered": 58.53019623299797, "cc_tiered_open": 60.211735029468755, "cc_tiered_sealed": 56.84865743652719, "low_n": false, "n": 300, "n_by_type": { "choice": 300 }, "score": 59.075407351269405 }, "noul": { "accuracy": 0.64, "calibration": 71.1232643568441, "cc": 28.000000000000004, "cc_by_type": { "noul": 28.000000000000004 }, "cc_tiered": 25.633881382349685, "cc_tiered_open": 29.155969634230512, "cc_tiered_sealed": 22.111793130468854, "low_n": false, "n": 150, "n_by_type": { "noul": 150 }, "score": 28.000000000000004 }, "score": { "accuracy": 0.5133333333333333, "calibration": 81.89277391203026, "cc": 42.517174243346886, "cc_by_type": { "score": 42.517174243346886 }, "cc_tiered": 39.453833031563924, "cc_tiered_open": 34.332851503658375, "cc_tiered_sealed": 44.57481455946947, "low_n": false, "n": 150, "n_by_type": { "score": 150 }, "score": 42.517174243346886 } }, "raw_on_A": { "calibration": 73.34529142011819, "capability": 57.27563081787769, "intelligence": 41.205970215637194 }, "set": "A2+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 300 }, "noul": { "invalid_rate": 0.0, "n": 150 }, "score": { "invalid_rate": 0.0, "n": 150 } }, "views": { "Balanced 33:33:33": 38.09522584926494, "Emphasis on Accuracy 60:20:20": 32.951561673121084, "Emphasis on Cost 20:20:60": 38.48641935280616, "Emphasis on Speed 20:60:20": 44.60450802438203, "Intelligence only": 27.401820493332366 }, "model_pin": null, "last_measured_on": null, "measurement_date_status": "unknown", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.74, "acc_among_decisive": 0.8648648648648649, "valid": 150 }, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "author submission" }, "class": "decision-api", "open": "no", "licence": "proprietary hosted API", "repo": "https://wity.alphanimble.com/", "endpoint_condition": "operator's hosted production API, measured as is; API default is off, our request sets reasoning=off; labelled configuration variant. The first approximately 70 decisions ran alongside two other streams; the remainder ran alone.", "underlying": "Qwen3.6-35B-A3B (operator self-reported; not independently verified)", "adapter_id": "run/wity-1-off.spec.json:wity_reasoning_adapter.WityReasoningAdapter", "api_exposure_note": "API measurement: the operator's endpoint received sealed item text, without answers.", "alt": { "label": "Qwen3.6-35B-A3B base-model reference price", "note": "Operator self-reported base, not independently verified; frozen Qwen/Qwen3.6-35B-A3B reference USD 0.15/M input and 1.00/M output, applied to the same measured tokens.", "usd_per_1000": 0.08721059113300492, "axes": { "calibration": 83.45289319036758, "cost": 41.78292309262718, "intelligence": 60.40492797149856, "speed": 87.66397534612332 } } } ], "not_measured": [ { "key": "autojev-27b-rtxpro6000-a2", "display": "AutoJev-27B (RTX PRO 6000)", "author": "unknown", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://huggingface.co/denis-pplx/autojev-27b" }, { "key": "bosun-v31-0.6b", "display": "Bosun v3.1 0.6B", "author": "Clause Logic", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://huggingface.co/Hanno-Labs/bosun-v3.1-0.6b" }, { "key": "classifier-dev-fast", "display": "classifier.dev (fast tier)", "author": "mrmps (@michael_chomsky)", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://classifier.dev" }, { "key": "decision-machine-1", "display": "decision-machine-1 (milliseconds.ai)", "author": "milliseconds.ai (Baptiste Laget)", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://www.milliseconds.ai" }, { "key": "deepseek-flash", "display": "DeepSeek V4.1 Flash (thinking default)", "author": "DeepSeek", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://benchmarkheaven.com/jev-models/deepseek-flash" }, { "key": "gemini-3.1-flash-lite", "display": "Gemini 3.1 Flash-Lite", "author": "Google", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://benchmarkheaven.com/jev-models/gemini-3.1-flash-lite" }, { "key": "gliner2", "display": "GLiNER2 (Fastino, gliner2.5-base)", "author": "Fastino", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/fastino-ai/GLiNER2" }, { "key": "gliner2-large", "display": "GLiNER2 large (Fastino)", "author": "Fastino", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://huggingface.co/fastino/gliner2-large-v1" }, { "key": "gliner2.5-multi", "display": "GLiNER2.5 multi (Fastino, 287M)", "author": "Fastino", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://huggingface.co/fastino/gliner2.5-multi-v1" }, { "key": "gliner2.5-small", "display": "GLiNER2.5 small (Fastino, 74M)", "author": "Fastino", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://huggingface.co/fastino/gliner2.5-small-v1" }, { "key": "gpt-5.6-luna", "display": "GPT-5.6 Luna (low reasoning effort)", "author": "OpenAI", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://benchmarkheaven.com/jev-models/gpt-5.6-luna" }, { "key": "gpt-6-luna", "display": "GPT-6 Luna (default medium reasoning effort)", "author": "OpenAI", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://benchmarkheaven.com/jev-models/gpt-6-luna" }, { "key": "gpt-6-luna-low", "display": "GPT-6 Luna (low reasoning effort)", "author": "OpenAI", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://benchmarkheaven.com/jev-models/gpt-6-luna-low" }, { "key": "instinct", "display": "Instinct (ZooWork, Qwen3.8-27B)", "author": "rayrain-srp (ZooWork)", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/fstandhartinger/jevbench/issues/69" }, { "key": "instinct-dual-4b", "display": "Instinct Dual 4B", "author": "ZooWork / pierre-srp", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/fstandhartinger/jevbench/issues/123" }, { "key": "jeff", "display": "jeff (Logan Markewich, GLiFormer 400M)", "author": "Logan Markewich", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/logan-markewich/jeff" }, { "key": "jevact", "display": "JevAct (einptein, jev1-2b-v2)", "author": "einptein", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/fstandhartinger/jevbench/issues/66" }, { "key": "jobe-qwen3.5-4b", "display": "Jobe Qwen3.5-4B (frozen)", "author": "MantisShrimpdev", "addendum": null, "status": "not measured", "reason": "No v1.6.0 score. Prior v1.5.6 catalogue: no official score was published.", "repo": "https://benchmarkheaven.com/jev-models/jobe-qwen3.5-4b" }, { "key": "kushal-gemma4-31b-it-autoloops", "display": "Autoloops – Gemma 4 31B IT", "author": "Autoloops", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://benchmarkheaven.com/jev-models/kushal-gemma4-31b-it-autoloops" }, { "key": "mica-v01-4b", "display": "mica-v01-4b", "author": "unknown", "addendum": { "id": "A1", "release": "v1.5", "label": "v1.5 roster addendum A1" }, "status": "not measured", "reason": "No v1.6.0 score. Prior v1.5.6 catalogue: The frozen refusal policy stopped the run after 1,088 of 1,624 rows: 27 documented refusals were mapped to HTTP 422, then three consecutive passthrough HTTP 400 refusals triggered exit 6. The remaining 536 rows have no scores, so this system is not eligible for an official rank.", "repo": "https://benchmarkheaven.com/jev-models/mica-v01-4b" }, { "key": "needle-3", "display": "Needle 3 (Cactus, 2-bit, local CPU)", "author": "Cactus Compute", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/cactus-compute/needle" }, { "key": "needle-3-tools", "display": "Needle 3, options as tools (post-hoc adapter mode)", "author": "Cactus Compute", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/cactus-compute/needle" }, { "key": "ninfer-qwen3.8-27b-t1.5", "display": "NInfer Qwen3.8-27B NVFP4 (T=1.5)", "author": "Igor L. / NInfer contributors", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/igorls/ninfer" }, { "key": "openjev-razorback16", "display": "OpenJev (DiffusionGemma 26B-A4B NVFP4, razorback16)", "author": "razorback16 / Codiv", "addendum": null, "status": "not measured", "reason": "No v1.6.0 score. Prior v1.5.6 catalogue: no official score was published.", "repo": "https://benchmarkheaven.com/jev-models/openjev-razorback16" }, { "key": "openjev-sglang", "display": "openjev-sglang (Qwen3.6-35B-A3B on SGLang)", "author": "ekzhang", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/ekzhang/openjev-sglang" }, { "key": "qwen3.8-27b", "display": "Qwen3.8 27B (Chutes TEE)", "author": "Qwen / Chutes", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://benchmarkheaven.com/jev-models/qwen3.8-27b" }, { "key": "reflex-27b", "display": "reflex-27b (Qwen3.8-27B)", "author": "kshetrajna12", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/kshetrajna12/reflex" }, { "key": "simplejev-qwen3.6-35b-a3b", "display": "SimpleJev Qwen3.6-35B-A3B", "author": "Featherless AI", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/featherless-ai/simple-jev" }, { "key": "simplejev-qwen3.8-27b", "display": "SimpleJev Qwen3.8-27B", "author": "Featherless AI", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/featherless-ai/simple-jev" }, { "key": "surogate-rune-26b-a4b-v3-rtxpro6000-a2", "display": "Surogate Rune 26B-A4B v3 (RTX PRO 6000)", "author": "unknown", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://huggingface.co/api/models/surogate/rune-26b-a4b-GGUF" }, { "key": "system-one-open", "display": "system-one-open (Gemma 4 E2B LoRA on an L4)", "author": "mithalouni", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/mithalouni/system-one-open" }, { "key": "vansa-3.4", "display": "Vansa-3.4 (Vansa, hosted System One API)", "author": "Vansa", "addendum": null, "status": "not measured", "reason": "Not measured on v1.6.0; the latest published score is shown separately with its date.", "repo": "https://vansa.org" }, { "key": "von-395m", "display": "Von (wfzyx, Option-Marker 395M)", "author": "wfzyx (Victor Hugo)", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/wfzyx/von" } ], "roster_count": 127, "overnight": { "round": "score-overnight-12", "scored_utc": "2026-10-05 09:06:16 UTC", "a2_note": "The original API subset A is retired after exposure to two providers. Newly measured hosted APIs use a fresh supplementary A2 draw of 300 sealed items (150 Choice, 75 Noul, 75 Score), disjoint from S and P, with the seed committed before the draw, plus the same public P300. A2 has reached four external entities, including hosting proxies, and is retired for future draws. API A2 scores are equated using the median S+P minus A2+P offset over 12 self-hosted systems: Intelligence -0.29, Calibration +1.56. That pool leans toward weaker systems; its main-A offset differs from the full-pool offset by about 1.8 Intelligence points, so strong API rows may have roughly ±2 points of equating bias. Bootstrap intervals cover pool resampling, not this selection effect. A2 topic and use-case radar cells cover P300 only because sealed A2 items have no topic labels; cells under 15 items are omitted. Family and language cells cover A2+P600. Sage 1.3.0 uses the fresh supplementary A3 subset: 300 never-exposed sealed items plus the same public P300, equated over 15 self-hosted systems (Intelligence offset +1.34, Calibration +3.72). A3 was sent to TypeSafe and Levanto and is now retired. Sage category and language cells cover public P300 only, with cells under 15 items omitted.", "notes": [ "Scores are the official v1.6.0 scorer (score_v16.py, method option B / O1S, bootstrap B = 1,000) over complete outputs only; each scoring round is kept separately.", "Hosted APIs are scored on their API subset plus P and equated to the self-hosted scale; under the exposure rule the v1.6.0 subset A is retired, so newly measured APIs use a fresh supplementary draw.", "GPU-class deviations: reproducible recipes used the hardware listed per row, including H100 for large fast-lane decoders and RTX 6000 for the baseline; hardware differences remain in measured latency. The standard x2 + 0.15 s self-hosted adjustment is an assumption, not a hardware normalization.", "Jev-class caps use a fixed reference: Jev 1.13.0 as measured in v1.5 (p50 0.62 s, USD 0.0323 per 1k answers); caps = 2x (1.23 s, USD 0.0646). Jev's own v1.6 p50 is 0.24 s. Wity auto remains outside the latency cap and stays ranked in Composite A. OFF and ALWAYS are unranked variants of the AUTO main row.", "6 further measured candidates await a separate publication decision.", "Sage 1.3.0 (Levanto Labs) was measured on 5 Oct from Sandy (Helsinki), text only. Cost uses the Levanto list tariff (USD 0.05/M input, USD 10/M output; levanto.ai/pricing, read 5 Oct) over all 600 answered rows: USD 0.0766/1,000 decisions. It exceeds the frozen Jev-class cost cap and is excluded from the Capability headline.", "A fresh Monday Jev 1.13.0 re-check on A3 measured p50 0.239 s and equated Capability 77.5 versus the official 76.5, within the confidence interval; the official v1.6.0 Jev row is retained." ], "exposure": { "wity-1-auto": { "display": "wity-1-auto (wity via railway)", "sealed_items_exposed": 300, "draws": "A2 (300 sealed) + P (300 public)", "note": "measured on v1.6.0 A2 u P" }, "wity-1-off": { "display": "wity-1-off (wity via railway)", "sealed_items_exposed": 300, "draws": "A2 (300 sealed) + P (300 public)", "note": "measured on v1.6.0 A2 u P" }, "wity-1-always": { "display": "wity-1-always (wity via railway)", "sealed_items_exposed": 300, "draws": "A2 (300 sealed) + P (300 public)", "note": "measured on v1.6.0 A2 u P" }, "jev-1.13.0": { "display": "jev-1.13.0 (typesafe)", "sealed_items_exposed": 600, "draws": "A (300 sealed, official) + A3 (300 sealed, re-check) + P (300 public)", "note": "official A measurement retained; 5 Oct A3 re-check confirmed it; A3 now retired" }, "fastino-gliner-2-5-decide": { "display": "fastino-gliner-2-5-decide (fastino)", "sealed_items_exposed": 300, "draws": "A (300 sealed) + P (300 public)", "note": "measured on v1.6.0 A u P" }, "gpt-6-luna": { "display": "gpt-6-luna (openai)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "gpt-6-luna-low": { "display": "gpt-6-luna-low (openai)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "gpt-5.6-luna": { "display": "gpt-5.6-luna (openai)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "gemini-3.1-flash-lite": { "display": "gemini-3.1-flash-lite (google)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "deepseek-flash": { "display": "deepseek-flash (deepseek)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "qwen3.8-27b": { "display": "qwen3.8-27b (chutes)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "decision-machine-1": { "display": "decision-machine-1 (milliseconds.ai)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "classifier-dev-fast": { "display": "classifier-dev-fast (classifier.dev)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "vansa-3.4": { "display": "vansa-3.4 (vansa)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "kushal-gemma4-31b-it-autoloops": { "display": "kushal-gemma4-31b-it-autoloops (autoloops)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "system-one-open": { "display": "system-one-open (modal via modal)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "openjev-sglang": { "display": "openjev-sglang (modal via modal)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "instinct": { "display": "instinct (zoowork)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "instinct-dual-4b": { "display": "instinct-dual-4b (zoowork)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "simplejev-qwen3.8-27b": { "display": "simplejev-qwen3.8-27b (featherless)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "simplejev-qwen3.6-35b-a3b": { "display": "simplejev-qwen3.6-35b-a3b (featherless)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "jevact": { "display": "jevact (jevact)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "sage-1.3.0": { "display": "Sage 1.3.0 (Levanto Labs)", "sealed_items_exposed": 300, "draws": "A3 (300 sealed) + P (300 public)", "note": "measured 5 Oct 2026; A3 exposed to TypeSafe and Levanto, now retired" } } } }