{ "G_med": 10.073244581339713, "G_med_api_basis_P_vs_A": 10.464452369024766, "G_med_flag_gt10": true, "benchmark": "JevBench", "board": { "A": { "markers": [], "order": [ "blink-v0-3-26b-a4b-nvfp4", "wald-4b-v2-1", "jiwo-4b", "rsi-jev-v6.1-vl-27b", "standard_one_8b_sh", "classone-gemma4-e4b", "classone-qwen3.5-4b", "liquid-d1-3b", "clef-omni", "jiwo-0.8b", "vegaml-4b", "gutsy-0.8b-v0.3", "classone-qwen3.5-2b", "liquid-d1-omni-600m", "vegaml-0.8b", "modernbert-jev-v2" ] }, "B": { "markers": [], "order": [ "blink-v0-3-26b-a4b-nvfp4", "wald-4b-v2-1", "jiwo-4b", "rsi-jev-v6.1-vl-27b", "standard_one_8b_sh", "classone-gemma4-e4b", "classone-qwen3.5-4b", "liquid-d1-3b", "clef-omni", "jiwo-0.8b", "vegaml-4b", "gutsy-0.8b-v0.3", "classone-qwen3.5-2b", "liquid-d1-omni-600m", "vegaml-0.8b", "modernbert-jev-v2" ] }, "C": { "markers": [], "order": [ "blink-v0-3-26b-a4b-nvfp4", "wald-4b-v2-1", "rsi-jev-v6.1-vl-27b", "jiwo-4b", "standard_one_8b_sh", "classone-gemma4-e4b", "classone-qwen3.5-4b", "liquid-d1-3b", "clef-omni", "jiwo-0.8b", "vegaml-4b", "gutsy-0.8b-v0.3", "classone-qwen3.5-2b", "liquid-d1-omni-600m", "vegaml-0.8b", "modernbert-jev-v2" ] }, "capability": { "order": [ "rsi-jev-v6.1-vl-27b", "blink-v0-3-26b-a4b-nvfp4", "wald-4b-v2-1", "jiwo-4b", "standard_one_8b_sh", "clef-omni", "liquid-d1-3b", "jiwo-0.8b", "gutsy-0.8b-v0.3", "classone-gemma4-e4b", "classone-qwen3.5-4b", "vegaml-4b", "modernbert-jev-v2", "vegaml-0.8b", "liquid-d1-omni-600m", "classone-qwen3.5-2b" ] } }, "bootstrap": { "B": 1000, "bootstrap_seed": 16, "g_med_fixed": true }, "headline": "A", "label": "JevBench v1.6.9 \u00b7 fresh regular draw, five self-hosted rows join the v1.6.8 field", "measurement_notes": [ "Refusals, errors and context-bound failures stay in the full 1,500 denominator. Every row kept its own package, model, runtime and context settings; nothing was re-run for presentation.", "NOT DIRECTLY COMPARABLE WITH THE LIVE v1.6.1 BOARD. This release is a different draw with its own field median: G_med = 10.073244581339713, frozen from the eight v1.6.5 members (Clef-omni, Blink, modernbert-jev v2 and the five v1.6.9 rows do not move it), against 2.573387642438244 on the live field. G_med_flag_gt10 is TRUE, which has not been true in an earlier published JevBench artifact. Concretely: the gap penalty is max(0, 1 - max(0, (gap - G_med) - 8)/100), so this field's threshold is G_med + 8 = 18.073 and no row here is penalised for its public-vs-sealed gap, whereas at the live field's median SIX of the sixteen would be - the two 4B rows this revision adds are among them. The difference was measured, not estimated, and is small: rsi-jev-v6.1-vl-27b Intelligence 61.564 would be 59.551, standard_one_8b_sh 34.465 would be 32.460, jiwo-4b 34.784 would be 33.997, classone-qwen3.5-4b 27.338 would be 25.413, liquid-d1-3b 22.139 would be 21.949, vegaml-4b 16.689 would be 16.673, and wald-4b-v2-1 (gap 9.478, below the live threshold 10.573), blink-v0-3-26b-a4b-nvfp4 (gap 8.161), modernbert-jev-v2 (gap 2.433), classone-gemma4-e4b (gap 10.263, also just below the live threshold), jiwo-0.8b (gap 9.322), classone-qwen3.5-2b (gap 2.157) and the other four (clef-omni, gutsy-0.8b-v0.3, liquid-d1-omni-600m, vegaml-0.8b) are unchanged. The direction is one-sided: this field flatters its own members.", "Cost: no item is excluded from the cost basis on this draw. Every row is priced from its own measured usage block against the frozen 25 Sep 2026 snapshot, or from a documented per-1,000 override, so the paid draw's public-counter-overflow exclusions have no analogue here. The cost normalizer C0 is USD 0.001 per 1,000 decisions with 30 cost points per decade. Every price on these rows is a clearly-labelled ESTIMATE; none is a bill any of these systems issued.", "gutsy-0.8b-v0.3: 29 of 1,500 items returned HTTP 500 \"llama_decode returned 1\" - the longest items, which llama.cpp cannot decode into the author's 8,192-token context. They are scored wrong, as the method requires. This is a crash class rather than a declared refusal. Its cost is input-only and marked *est.: the author's server emits no output-token field, and under the 9 Oct 2026 rule a missing output_tokens on a model with no generation path at all is treated as 0 output tokens.", "standard_one_8b_sh and rsi-jev-v6.1-vl-27b: the same 29 longest items exceed each system's declared 32,768-token context and were refused with HTTP 422 rather than truncated; scored wrong per method. For RSI-Jev the refusal boundary was proven before the scored run: 13 of 13 boundary checks passed, including that a 32,768-token question is accepted and seen whole and a 32,769-token one returns 422 with nothing silently cut.", "liquid-d1-omni-600m truncates states on the right at 16,384 tokens by design, and its published run is the second of two: a first run on a 24 GB card hit 29 CUDA out-of-memory errors, which was our under-provisioning and not model behaviour; it is retained and not scored. vegaml-0.8b and vegaml-4b truncate 29 states on the right at 73,728 tokens by design. The Vega checkpoint repository declares no licence field; the vegaml package is Apache-2.0.", "v1.6.9 ADDS FIVE ROWS - jiwo 0.8b, jiwo 4b, ClassOne Qwen 3.5 2B, ClassOne Qwen 3.5 4B and ClassOne Gemma 4 E4B - to the eleven of v1.6.8 on the same draw, under the same rule: the field median stays the frozen v1.6.5 value, so all eleven earlier rows keep identical point scores, axes and capability; ranks and bootstrap confidence intervals are recomputed for the sixteen-row roster, and raw_on_A of standard_one_8b_sh and the field-level diagnostics (G_med_api_basis_P_vs_A, the API-basis offsets) move again for the reason given below; none of them enters a score. All five were measured in one tranche on ONE RTX 5090 32 GB pod in Brasov, Romania, strictly one row at a time, under a pre-registration written before the pod existed (r90/PREREG-R90.md, sha256 758959702bf04a3b8ab6a29f87d3320b2ca487b8548398341a37e2d3bf37f35d), on the authority of board #17148.", "jiwo 0.8b and jiwo 4b: the author's own package and `jiwo serve` at the pinned snapshot directory, because their server takes no revision and a bare repo id would resolve main. JIWO_MAX_LENGTH=65536 is the author's own documented value for their published runs; the 8,192 default refuses longer prompts with HTTP 400 and never truncates. 29 of 1,500 items - the longest states of this draw, 76,381 to 80,883 tokens as the author's own tokenizer counts them - exceed even 65,536 and were refused with HTTP 400; they are scored wrong, as the method requires (the pre-registration had expected 65,536 to cover every item, so this is a disclosed miss of that prediction, not a changed setting), and they are exactly the same 29 items the published gutsy, standard_one_8b_sh, RSI-Jev, Blink and ClassOne rows of this draw also lose to their own context limits. The trained context is 8,192 tokens, so every longer item is answered outside the range the model saw in training. CUDA graphs are OFF: the author's own card says graphs agree with eager on only 99.7 % of fields, so eager is the exact path - the author's 14.5 ms headline uses a mode this measurement does not use. Native probabilities, output tokens a literal 0, cost a labelled estimate from the author's own declared base. Speed: the author's install line does not install the optional causal_conv1d kernel, so transformers used its reference PyTorch fallback (stated in the serve logs; correct but slower). The same holds for the two ClassOne Qwen rows, not for ClassOne Gemma 4 E4B. The Speed axis of those four rows is that of the authors' stacks as shipped. Not comparable with the author's headline numbers, which are Decision Index 0.3, a DIFFERENT benchmark: \"1st of 27 under 1B\" and \"3rd of 26 in 3-6B\" are placements on that board. The jiwo 4b submitter's note claims \"top2 for its model size\", which contradicts the author's own card (3rd of 26 in the 3-6B class); and both cards' figures are v1's, while the measured revision is v2, whose placement the cards themselves mark pending. The authors publish no decontamination procedure against JevBench items; no JevBench item is involved either way, because the model is run here. English only by the author's own statement.", "ClassOne Qwen 3.5 2B, Qwen 3.5 4B and Gemma 4 E4B: the author's own install and serve lines at code pin 6cca132b, with CLASSONE_BASE_MODEL pointed at each pinned snapshot directory, which their own code accepts and which makes classone_heads.pt resolve locally, so no revision can drift and no request-time hub fetch exists. Their hard-coded MAX_TOKENS_PER_REQUEST = 8192 refuses longer prompts with HTTP 400 and never truncates: 29 of 1,500 items were refused that way (the same 29 longest states: 76,324 to 80,861 tokens as the author's own tokenizer counts them for the two Qwen rows, 76,378 to 82,123 for Gemma 4 E4B), so each ClassOne row answered 1,471. The pre-registration had predicted 23 such refusals, the count on the predecessor draw; this draw has 29, which is that prediction missing, not a changed setting. Those items are scored wrong, as the method requires, exactly as the two already-published ClassOne rows carry it. ONE documented mapping of ours and the only one: their server keys score probabilities \"1\"..\"n\" while JevBench's canonical keys are \"0\"..\"n-1\", so run 58's byte-identical ClassOneReindexAdapter relabels score keys k -> k-1 by calling the unchanged vendored typesafe adapter's own run(), and an item whose key set is not exactly {\"1\"..\"n\"} is refused rather than rescored on a guess; choice and noul are untouched. The same stricter run-58 smoke gate passed on synthetic requests before any sealed item was sent. Native probabilities, output tokens a literal 0. Not comparable with the author's public scorecard, which is self-reported on a different 231-item public set - and in the scorecard of the author's class-one code-repo README the Gemma 4 E4B row is byte-identical to the E2B row in all four of its JevBench columns, so at least one of those two is not a separate measurement. Cost for the two Qwen rows resolves through the unmodified frozen path at the author's own declared spelling; for Gemma 4 E4B pricing_v15.reference() raises on every spelling of the base, so that row carries a labelled estimate from the reference the frozen 25 Sep 2026 snapshot itself contains for exactly this base and size (deepinfra:google/gemma-4-E4B-it, USD 0.02 in / 0.10 out per 1M) - the same rate the published classone-gemma4-e2b row uses as a one-size-UP estimate. A release-lane line is recorded with that row: the base needs a listing decision or an UNLISTED_BASE_MODELS entry in the frozen table.", "v1.6.8 ADDED ONE ROW, modernbert-jev v2, to the ten of v1.6.7 on the same draw, under the same rule: the field median stays the frozen v1.6.5 value, so all ten earlier rows keep identical point scores, axes and capability; ranks and bootstrap confidence intervals were recomputed for the eleven-row roster, and raw_on_A of standard_one_8b_sh and the field-level diagnostics (G_med_api_basis_P_vs_A, the API-basis offsets) moved for the reason given below; none of them enters a score.", "modernbert-jev-v2: a 150M ModernBERT-base NLI encoder, int8 (unsigned) ONNX, served by the author's own CLI at its own defaults on ONE CPU thread of the evaluator's shared Sandy host (AMD Ryzen 5 3600, Zen 2), not a dedicated machine; the 1-minute load average was logged every 60 s and ranged about 5.5 to 27.9 (median 10.2) during the run, so Speed is a lower bound for this model on an idle core. The run was interrupted once by the evaluator's session ending after 41 answered items and resumed with the unchanged runner's --resume, which keeps answered rows and sends only the rest; the server is deterministic (byte-identical answers on repeated requests). By the author's documented design (README: \"windowing retrieval is future work\"), a state is cut at 20,000 characters and read through at most six 300-word windows spread across it: the 29 longest states of this draw are answered from that sample rather than refused, so words between the windows are never seen. Probabilities are native (per-head temperatures from the shipped calibration.json). With the shipped Noul temperature of 2.5, every one of its 375 Noul answers lies between P(yes) 0.23 and 0.80, so none counts as decisive under the O1S Noul method, which puts every Noul tier of this row at its floor; this is the model as shipped, not a serving fault. Output tokens are a literal 0; cost is a labelled estimate at the base-size encoder rate.", "v1.6.7 ADDED ONE ROW, Blink v0.3 26B-A4B, to the nine of v1.6.6 on the same draw, under the same rule: the field median stays the frozen v1.6.5 value, so all nine earlier rows keep identical point scores, axes and capability; ranks and bootstrap confidence intervals are recomputed for the ten-row roster, and raw_on_A of standard_one_8b_sh moves again for the reason given below. Blink's gap (8.161) is below the threshold.", "blink-v0-3-26b-a4b-nvfp4: measured from the model card's own plain-vLLM 0.30.0 serve command (NVFP4, fp8 KV cache, 32,768-token context, T=1.3 option-letter readout through a loopback shim reproducing the card's decide() client) with ONE serving-only deviation: VLLM_BATCH_INVARIANT=1. With the command as published, four identical repeats of 50 public items differed by up to 0.126 in probability, failing the pre-registered repeatability gate (0.005); with batch-invariant kernels the repeats were byte-identical. This makes the run repeatable, not more correct: between serving settings the same item's probabilities differ by up to 0.39 and the argmax by 2 of 49 public items, so a different stack or card can score differently. Latency is of the batch-invariant path (about twice the default per request). Run on an RTX 5090 32 GB (no RTX PRO 6000 was free). 29 of 1,500 states (77,706 to 82,401 tokens) exceed the card's 32,768-token context and were refused with HTTP 422; scored wrong per method. Cost is a labelled estimate from the base model google/gemma-4-26b-a4b-it (USD 0.09 in / 0.30 out per 1M, frozen 25 Sep 2026 OpenRouter snapshot).", "v1.6.6 ADDED ONE ROW, Clef-omni, to the eight of v1.6.5 on the same draw. The field median is held at the frozen v1.6.5 value (g_med_fixed, as for earlier same-draw additions), so all eight earlier rows keep identical point scores, axes and capability. Ranks and bootstrap confidence intervals are recomputed for the nine-row roster, as the v1.6.3 and v1.6.4 addenda did. One diagnostic also moves: raw_on_A (scores on the 300-item A subset) of standard_one_8b_sh, because the A-basis median is not part of the frozen value. Clef-omni's gap (5.555) is below the threshold.", "clef-omni: measured as shipped on its own in-process entry point at the pinned revision, defaults unchanged. 29 of 1,500 items reach the model's own 64,000-token default and are answered on the truncated state, as the authors' encoder does by design; none was refused. The Omni video processor needs torchvision to load, which the model card's tested stack does not list; torchvision 0.26.0+cu130, the build matching its torch 2.11.0+cu130, was added and nothing else changed. Output tokens are a literal 0, so the cost is input-only.", "wald-4b-v2-1: the shipped v2.1 server applies a documented Noul decisive floor (serving.json noul_decisive_floor 0.81): every calibrated P(yes) inside (0.19, 0.81) is moved to the nearer edge, on every Noul answer and not keyed to any item, so it answers all 375 Noul items decisively. Measured as shipped, server default effort none (one pass, 0 output tokens). It is the successor of the published wald-4b-v2 row on the paid draw and carries the same base-model cost basis.", "Subject topics and TypeSafe use cases were labelled for this draw's 1,500 items in their own run (Winnow-12B Q8 on our own pod, egress cut before the sealed upload, the unchanged T1-T3/U1/U2 rules; no new rule was added). On 75 held-out public items checked by hand, topic agreement after the rules is 71/75 = 94.7 %. Known systematic miss: 17 of 27 router_policy items are filed under Coding by the subject of the request rather than under Rules, policy & law by the routing task; left as labelled because adding a rule after seeing the sample would change the method." ], "n_ranked": 16, "not_measured": [], "noul_method": { "applied": "O1S", "native_method_source_sha256": "119e59486ab3300b972fbcbadb0fe1dceceb124287607375aa61c66063c607d0" }, "options": { "A": { "intelligence_floor": 50.0, "weights": { "calibration": 25, "cost": 25, "intelligence": 25, "speed": 25 } }, "B": { "intelligence_floor": 50.0, "weights": { "calibration": 20, "cost": 20, "intelligence": 40, "speed": 20 } }, "C": { "intelligence_floor": 60.0, "weights": { "calibration": 25, "cost": 25, "intelligence": 25, "speed": 25 } } }, "protocol": "jevbench::v1.6", "provisional": false, "revision": "v1.6.9", "roster_count": 16, "run_kind": "regular-queue", "sample": { "open": 300, "published_open": 300, "sealed": 1200, "total": 1500 }, "sealed_share_of_intelligence": 0.5, "source_note": "Sixteen systems: the five rows added in v1.6.9 (jiwo 0.8b, jiwo 4b, ClassOne Qwen 3.5 2B, ClassOne Qwen 3.5 4B, ClassOne Gemma 4 E4B), modernbert-jev v2, added in v1.6.8, Blink v0.3 26B-A4B, added in v1.6.7, Clef-omni, added in v1.6.6, and the eight v1.6.5 systems, all measured on the fresh regular draw v1.6-regular-20261010-a2 (seeded, rotation-ledger draw of 1,200 sealed + the 300-item epoch public set; every drawn item had zero prior scored uses and the set does not overlap v1.6.0 or the paid fast-lane draw). One score_v16.py invocation, unmodified frozen path, JEV_NOUL_METHOD=O1S, B=1000, seed 16. v1.6.9 is one score_v16 invocation over all sixteen with --g-med-fixed 10.073244581339713, the frozen v1.6.5 median, so the eleven earlier rows keep every point score. Drawn, frozen and scored by jevbench-add-requests-20260919 as v1.6.5 release owner (board #16539, #16625, #16630, #16703, #16809; v1.6.6 under CoS #16771, board #16827; v1.6.7 board #16903; v1.6.8 board #16971; v1.6.9 board #17147/#17148).", "source_sha256": "d947b30242403455f51e103f2b9a55193f174281d6944144a3f0c9355f2a9a29", "status": "published", "systems": [ { "addendum": null, "api_flag": false, "api_subset": false, "author": "Pixilab", "axes": { "calibration": 90.67498392881862, "cost": 49.62845783533821, "intelligence": 49.21581563112197, "speed": 91.48151024194223 }, "calibration": { "parts": { "choice": { "ece_hard": 0.04822184846568736, "ece_score": 90.35563030686254, "mean_tvd": null, "n": 739, "n_ece_hard": 317, "n_tvd": 0, "score": 90.35563030686254 }, "noul": { "abstention_rate": 0.3296703296703297, "brier": 0.10252744982113196, "ece": 0.049136862184262595, "n": 364, "score": 90.17262756314747 }, "score": { "interval90_coverage": 0.9483695652173914, "interval90_width": 0.3299365942028985, "n": 368, "nrps": 0.07594938571288598, "score": 91.49669391644588, "top_ece": 0.047058367979098334 } }, "score": 90.67498392881862 }, "capability": 69.9453997799703, "composite_ci95": { "A": [ 45.23103654293889, 64.78227929319183 ], "B": [ 41.83419561295226, 62.07842151813806 ], "C": [ 31.410442043707555, 52.48195783546016 ] }, "cost": { "basis": "ESTIMATE (*est.), base-model reference. v1.5-M2 base-model reference: Blink v0.3 26B-A4B is a full fine-tune of google/gemma-4-26B-A4B-it quantised to NVFP4 (ModelOpt); the base resolves in pricing_v15 as base_model google/gemma-4-26b-a4b-it -> OpenRouter google/gemma-4-26b-a4b-it, USD 0.09/M input, USD 0.30/M output, from the OpenRouter catalog snapshot 2026-09-25 (jevbench-v15-measure-20260925/receipts/OPENROUTER-MODELS-20260925.json, sha256 f5d0cd8e0f7d2d6514748c99bfcaeacc09775bac7aa7eecf765af4ba04b0dcd5), read by the frozen pricing_v15 on 2026-10-10. Tokens: input = vLLM prompt_tokens, output = 1 per answered question (one decode step, the option letter); both priced. The 29 context-refused decisions carry no tokens and stay in the 1,500-decision denominator.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.04775858 }, "display": "Blink v0.3 26B-A4B", "endpoint_condition": "evaluator-owned Lium RTX 5090 32 GB pod (Drobeta-Turnu Severin, Romania), offline after setup (egress cut and proven), plain vLLM 0.30.0 with the model card's own serve command plus ONE serving-only deviation: VLLM_BATCH_INVARIANT=1 (needed to pass the 0.005 repeatability gate; see DETERMINISM.md), served through our loopback shim implementing the card's reference decide() client (T=1.3, softmax over option-letter logprobs); context 32,768 (29 of 1,500 states are 77k-82k tokens and were refused by vLLM = recorded misses)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 53.29629960522241, "I_sealed": 45.135331657021524, "base": 49.21581563112197, "excess": -1.9122766331388288, "gap": 8.160967948200884, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 74.01883434266433, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 100.0, "hard": 67.9144385026738, "judge": 76.26895187870798, "standard": 69.8618668899121 } }, "open|noul": { "cc": 28.205659075224297, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 30.434782608695656, "judge": 0.0, "standard": 44.44444444444444 } }, "open|score": { "cc": 57.66440539777861, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 66.90386806087209, "hard": 59.08254746680842, "judge": 54.11814783561007, "standard": 55.527776271425026 } }, "sealed|choice": { "cc": 68.42011936132744, "n": { "easy": 72, "hard": 260, "judge": 141, "standard": 127 }, "tiers": { "easy": 83.10564188132234, "hard": 59.93184566340551, "judge": 68.95134696362436, "standard": 77.25706409372847 } }, "sealed|noul": { "cc": 20.840080971659916, "n": { "easy": 39, "hard": 132, "judge": 72, "standard": 57 }, "tiers": { "easy": 64.1025641025641, "hard": 0.0, "judge": 30.555555555555557, "standard": 26.315789473684205 } }, "sealed|score": { "cc": 46.145794638077234, "n": { "easy": 33, "hard": 115, "judge": 75, "standard": 77 }, "tiers": { "easy": 59.269920865495806, "hard": 31.19109112767333, "judge": 57.63492575992532, "standard": 52.25944186240362 } } } }, "jevbench_score": 61.16059214787521, "key": "blink-v0-3-26b-a4b-nvfp4", "last_measured_on": "2026-10-10", "licence": "Apache-2.0 (card tag; no LICENSE file in the repo); base google/gemma-4-26B-A4B-it Apache-2.0", "listing": "ranked", "measured_in": "v1.6.7", "measurement_date_status": "measured", "model_pin": "PixilabAI/Blink-v0.3-26B-A4B-NVFP4@08caad20c1574fb76341ca809ff38fdb081f7d4a", "noul_decisive": { "acc_among_decisive": 0.9467213114754098, "decisive": 244, "decisive_rate": 0.6703296703296703, "n": 375, "supported": true, "valid": 364 }, "planned_weight_denominator": 1.0, "rank": 1, "ranked": true, "ranks": { "A": 1, "B": 1, "C": 1, "capability": 2 }, "repo": "https://huggingface.co/PixilabAI/Blink-v0.3-26B-A4B-NVFP4", "scores": { "A": 61.16059214787521, "B": 57.67804533002703, "C": 42.47263343602446 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.252750841062516, "p50_s_raw": 0.05137542053125799, "p95_s_adjusted": 0.2812913428759202, "p95_s_raw": 0.06564567143796012 }, "status": { "answered_ok": 1471, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns": { "family": { "calibration": { "accuracy": 0.8620689655172413, "cc": 78.57219756866152, "cc_by_type": { "choice": 91.66666666666667, "noul": 66.66666666666667, "score": 77.38325937265121 }, "low_n": true, "n": 29, "n_by_type": { "choice": 16, "noul": 6, "score": 7 }, "score": 78.57219756866152 }, "calibration_gen": { "accuracy": 0.6607142857142857, "cc": 44.41501762938165, "cc_by_type": { "choice": 69.72049689440995, "noul": 15.384615384615374, "score": 48.139940609119634 }, "low_n": false, "n": 112, "n_by_type": { "choice": 56, "noul": 26, "score": 30 }, "score": 44.41501762938165 }, "composite_gen": { "accuracy": 0.7755102040816326, "cc": 61.661977649287216, "cc_by_type": { "choice": 81.48148148148148, "noul": 28.57142857142858, "score": 74.93302289495159 }, "low_n": false, "n": 98, "n_by_type": { "choice": 36, "noul": 28, "score": 34 }, "score": 61.661977649287216 }, "fairness": { "accuracy": 1.0, "cc": 98.74247115687324, "cc_by_type": { "choice": 100.0, "noul": 100.0, "score": 96.22741347061967 }, "low_n": true, "n": 27, "n_by_type": { "choice": 12, "noul": 6, "score": 9 }, "score": 98.74247115687324 }, "jaggedness": { "accuracy": 0.5243243243243243, "cc": 12.830160086648034, "cc_by_type": { "choice": 55.53496989346919, "noul": -58.97435897435898, "score": 41.92986934083389 }, "low_n": false, "n": 185, "n_by_type": { "choice": 97, "noul": 39, "score": 49 }, "score": 12.830160086648034 }, "llm_router": { "accuracy": 0.7992424242424242, "cc": 72.61663013184426, "cc_by_type": { "choice": 65.62935438922435, "noul": 75.34246575342465, "score": 76.87807025288377 }, "low_n": false, "n": 264, "n_by_type": { "choice": 131, "noul": 73, "score": 60 }, "score": 72.61663013184426 }, "long_state": { "accuracy": 0.0, "cc": -110.31746031746032, "cc_by_type": { "choice": -30.952380952380945, "noul": -100.0, "score": -200.0 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 11, "score": 7 }, "score": 0.0 }, "ml_usecase": { "accuracy": 0.4838709677419355, "cc": 21.95983819644401, "cc_by_type": { "choice": 35.38461538461538, "noul": 0.0, "score": 30.494899204716653 }, "low_n": false, "n": 31, "n_by_type": { "choice": 15, "noul": 8, "score": 8 }, "score": 21.95983819644401 }, "multilingual": { "accuracy": 0.6984126984126984, "cc": 49.44236254308296, "cc_by_type": { "choice": 72.2346618898343, "noul": 26.436781609195403, "score": 49.655644130219166 }, "low_n": false, "n": 315, "n_by_type": { "choice": 146, "noul": 87, "score": 82 }, "score": 49.44236254308296 }, "negation_pair_gen": { "accuracy": 0.7321428571428571, "cc": 43.01413459029695, "cc_by_type": { "choice": 90.47619047619048, "noul": -9.999999999999998, "score": 48.56621329470037 }, "low_n": false, "n": 56, "n_by_type": { "choice": 28, "noul": 20, "score": 8 }, "score": 43.01413459029695 }, "option_order": { "accuracy": 0.9649122807017544, "cc": 95.5604883462819, "cc_by_type": { "choice": 95.5604883462819 }, "low_n": false, "n": 57, "n_by_type": { "choice": 57 }, "score": 95.5604883462819 }, "perturb_pair": { "accuracy": 0.9, "cc": 84.88006962790548, "cc_by_type": { "choice": 84.27260812581913, "noul": 85.71428571428572, "score": 84.6533150436116 }, "low_n": false, "n": 60, "n_by_type": { "choice": 34, "noul": 14, "score": 12 }, "score": 84.88006962790548 }, "usecase": { "accuracy": 0.6286919831223629, "cc": 38.36326470609512, "cc_by_type": { "choice": 67.6238196184236, "noul": 19.298245614035082, "score": 28.167728885826694 }, "low_n": false, "n": 237, "n_by_type": { "choice": 111, "noul": 57, "score": 69 }, "score": 38.36326470609512 } }, "lang": { "ar": { "accuracy": 0.5454545454545454, "cc": 4.937825248418551, "cc_by_type": { "choice": 20.000000000000004, "noul": 0.0, "score": -5.186524254744351 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 4, "score": 2 }, "score": 4.937825248418551 }, "cs": { "accuracy": 0.75, "cc": 53.954802259887, "cc_by_type": { "choice": 74.57627118644068, "noul": 33.33333333333333 }, "low_n": true, "n": 8, "n_by_type": { "choice": 5, "noul": 3 }, "score": 53.954802259887 }, "da": { "accuracy": 0.7916666666666666, "cc": 54.43977410930783, "cc_by_type": { "choice": 64.49704142011835, "noul": 19.999999999999996, "score": 78.82228090780514 }, "low_n": true, "n": 24, "n_by_type": { "choice": 11, "noul": 5, "score": 8 }, "score": 54.43977410930783 }, "de": { "accuracy": 0.8, "cc": 70.03622189564423, "cc_by_type": { "choice": 83.70549185274592, "noul": 77.77777777777777, "score": 48.625396056408995 }, "low_n": false, "n": 45, "n_by_type": { "choice": 24, "noul": 9, "score": 12 }, "score": 70.03622189564423 }, "el": { "accuracy": 0.4444444444444444, "cc": -4.511613763017206, "cc_by_type": { "choice": 51.80722891566265, "noul": -60.0, "score": -5.3420702047142665 }, "low_n": true, "n": 18, "n_by_type": { "choice": 11, "noul": 5, "score": 2 }, "score": 0.0 }, "en": { "accuracy": 0.6979445933869526, "cc": 46.31123906153604, "cc_by_type": { "choice": 70.16264722378011, "noul": 19.548872180451138, "score": 49.222197780376874 }, "low_n": false, "n": 1119, "n_by_type": { "choice": 577, "noul": 266, "score": 276 }, "score": 46.31123906153604 }, "es": { "accuracy": 0.6785714285714286, "cc": 54.13125875630315, "cc_by_type": { "choice": 47.63636363636363, "noul": 63.63636363636365, "score": 51.12104899618216 }, "low_n": true, "n": 28, "n_by_type": { "choice": 10, "noul": 11, "score": 7 }, "score": 54.13125875630315 }, "fi": { "accuracy": 0.3333333333333333, "cc": -8.920187793427232, "cc_by_type": { "choice": 15.49295774647887, "noul": -33.333333333333336 }, "low_n": true, "n": 9, "n_by_type": { "choice": 6, "noul": 3 }, "score": 0.0 }, "fr": { "accuracy": 0.7333333333333333, "cc": 39.085828039995185, "cc_by_type": { "choice": 100.00000000000001, "noul": -50.0, "score": 67.25748411998555 }, "low_n": true, "n": 15, "n_by_type": { "choice": 5, "noul": 4, "score": 6 }, "score": 39.085828039995185 }, "hi": { "accuracy": 0.5333333333333333, "cc": 38.75519442187261, "cc_by_type": { "choice": 50.31055900621117, "noul": 19.999999999999996, "score": 45.95502425940667 }, "low_n": false, "n": 30, "n_by_type": { "choice": 11, "noul": 10, "score": 9 }, "score": 38.75519442187261 }, "id": { "accuracy": 0.6666666666666666, "cc": 33.87096774193548, "cc_by_type": { "choice": 67.74193548387096, "noul": 0.0 }, "low_n": true, "n": 6, "n_by_type": { "choice": 4, "noul": 2 }, "score": 33.87096774193548 }, "it": { "accuracy": 0.8823529411764706, "cc": 76.94126084433556, "cc_by_type": { "choice": 100.00000000000001, "noul": 100.0, "score": 30.82378253300666 }, "low_n": true, "n": 17, "n_by_type": { "choice": 5, "noul": 4, "score": 8 }, "score": 76.94126084433556 }, "ja": { "accuracy": 0.53125, "cc": 25.083725989475457, "cc_by_type": { "choice": 59.34959349593497, "noul": -14.28571428571429, "score": 30.187298758205692 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 25.083725989475457 }, "ko": { "accuracy": 0.7272727272727273, "cc": 74.31438742885162, "cc_by_type": { "choice": 100.0, "noul": 100.0, "score": 22.943162286554863 }, "low_n": true, "n": 11, "n_by_type": { "choice": 4, "noul": 2, "score": 5 }, "score": 74.31438742885162 }, "mixed": { "accuracy": 1.0, "cc": 99.95012548973524, "cc_by_type": { "choice": 100.0, "noul": 100.0, "score": 99.85037646920576 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 4, "score": 1 }, "score": 99.95012548973524 }, "nl": { "accuracy": 0.6923076923076923, "cc": 21.532452114485192, "cc_by_type": { "choice": 100.0, "noul": 33.33333333333333, "score": -68.73597698987774 }, "low_n": true, "n": 13, "n_by_type": { "choice": 5, "noul": 6, "score": 2 }, "score": 21.532452114485192 }, "no": { "accuracy": 0.5714285714285714, "cc": 33.87978142076503, "cc_by_type": { "choice": 34.42622950819673, "noul": 33.33333333333333 }, "low_n": true, "n": 7, "n_by_type": { "choice": 4, "noul": 3 }, "score": 33.87978142076503 }, "pl": { "accuracy": 0.65, "cc": 44.580498742442956, "cc_by_type": { "choice": 92.12942719720158, "noul": 0.0, "score": 41.612069030127294 }, "low_n": false, "n": 40, "n_by_type": { "choice": 16, "noul": 10, "score": 14 }, "score": 44.580498742442956 }, "pt": { "accuracy": 0.9444444444444444, "cc": 90.22683698066082, "cc_by_type": { "choice": 81.90954773869345, "noul": 100.0, "score": 88.77096320328903 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 6, "score": 5 }, "score": 90.22683698066082 }, "sv": { "accuracy": 0.625, "cc": 36.356054299978204, "cc_by_type": { "choice": 55.55555555555555, "noul": 0.0, "score": 53.51260734437906 }, "low_n": true, "n": 8, "n_by_type": { "choice": 3, "noul": 4, "score": 1 }, "score": 36.356054299978204 }, "tr": { "accuracy": 0.8181818181818182, "cc": 60.31616533251732, "cc_by_type": { "choice": 73.6842105263158, "noul": 100.0, "score": 7.264285471236143 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 3, "score": 3 }, "score": 60.31616533251732 }, "uk": { "accuracy": 0.6363636363636364, "cc": 10.754585109986635, "cc_by_type": { "choice": 56.52173913043478, "noul": -100.0, "score": 75.74201619952514 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 1, "score": 4 }, "score": 10.754585109986635 }, "zh": { "accuracy": 0.875, "cc": 62.19801371486293, "cc_by_type": { "choice": 100.00000000000001, "noul": 100.0, "score": -13.405958855411203 }, "low_n": true, "n": 8, "n_by_type": { "choice": 4, "noul": 3, "score": 1 }, "score": 62.19801371486293 } }, "type": { "choice": { "accuracy": 0.7693333333333333, "cc": 70.11573674209708, "cc_by_type": { "choice": 70.11573674209708 }, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 70.11573674209708 }, "noul": { "accuracy": 0.616, "cc": 23.2, "cc_by_type": { "noul": 23.2 }, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 23.2 }, "score": { "accuracy": 0.6293333333333333, "cc": 48.580321460438455, "cc_by_type": { "score": 48.580321460438455 }, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 48.580321460438455 } }, "usecase": { "advertising": { "accuracy": 0.75, "cc": 43.75694683730911, "cc_by_type": { "choice": 70.34596375617794, "noul": 19.999999999999996, "score": 40.924876755749395 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 43.75694683730911 }, "code_linting": { "accuracy": 0.6666666666666666, "cc": 30.4794447005702, "cc_by_type": { "choice": 70.54009819967267, "noul": 19.999999999999996, "score": 0.8982359020379249 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 30.4794447005702 }, "customer_support": { "accuracy": 0.65, "cc": 47.88048103401948, "cc_by_type": { "choice": 67.74193548387096, "noul": 25.0, "score": 50.89950761818749 }, "low_n": true, "n": 20, "n_by_type": { "choice": 8, "noul": 8, "score": 4 }, "score": 47.88048103401948 }, "demand_forecasting": { "accuracy": 0.19230769230769232, "cc": -40.46675998594265, "cc_by_type": { "choice": 3.614457831325295, "noul": -100.0, "score": -25.01473778915324 }, "low_n": true, "n": 26, "n_by_type": { "choice": 11, "noul": 4, "score": 11 }, "score": 0.0 }, "ecommerce_marketplaces": { "accuracy": 0.5, "cc": 30.134531521647336, "cc_by_type": { "choice": 62.73291925465839, "noul": -19.999999999999996, "score": 47.670675310283606 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 5, "score": 6 }, "score": 30.134531521647336 }, "feature_extraction": { "accuracy": 0.5862068965517241, "cc": 40.83135316808231, "cc_by_type": { "choice": 65.81196581196579, "noul": 19.999999999999996, "score": 36.68209369228115 }, "low_n": true, "n": 29, "n_by_type": { "choice": 15, "noul": 5, "score": 9 }, "score": 40.83135316808231 }, "financial_crime": { "accuracy": 0.6153846153846154, "cc": 43.836506452390005, "cc_by_type": { "choice": 73.91304347826087, "noul": 25.0, "score": 32.59647587890912 }, "low_n": true, "n": 26, "n_by_type": { "choice": 10, "noul": 8, "score": 8 }, "score": 43.836506452390005 }, "gaming": { "accuracy": 0.5517241379310345, "cc": 13.446554121042317, "cc_by_type": { "choice": 52.475247524752476, "noul": -14.28571428571429, "score": 2.150129124088762 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 7, "score": 11 }, "score": 13.446554121042317 }, "insurance_claims": { "accuracy": 0.5, "cc": 39.067485028605155, "cc_by_type": { "choice": 20.634920634920636, "noul": 19.999999999999996, "score": 76.56753445089484 }, "low_n": true, "n": 14, "n_by_type": { "choice": 8, "noul": 5, "score": 1 }, "score": 39.067485028605155 }, "knowledge_graphs": { "accuracy": 0.5652173913043478, "cc": 30.3227658858092, "cc_by_type": { "choice": 54.198473282442755, "noul": 19.999999999999996, "score": 16.769824374984854 }, "low_n": true, "n": 23, "n_by_type": { "choice": 14, "noul": 5, "score": 4 }, "score": 30.3227658858092 }, "lead_generation": { "accuracy": 0.48, "cc": 30.591986352709828, "cc_by_type": { "choice": 83.73983739837398, "noul": 0.0, "score": 8.036121659755512 }, "low_n": true, "n": 25, "n_by_type": { "choice": 8, "noul": 8, "score": 9 }, "score": 30.591986352709828 }, "legal_compliance": { "accuracy": 0.6153846153846154, "cc": 61.8756462123043, "cc_by_type": { "choice": 67.21311475409836, "noul": 100.0, "score": 18.41382388281453 }, "low_n": true, "n": 13, "n_by_type": { "choice": 8, "noul": 1, "score": 4 }, "score": 61.8756462123043 }, "llm_guardrails": { "accuracy": 0.7931034482758621, "cc": 60.58510134359349, "cc_by_type": { "choice": 81.65137614678899, "noul": 25.0, "score": 75.10392788399149 }, "low_n": true, "n": 29, "n_by_type": { "choice": 14, "noul": 8, "score": 7 }, "score": 60.58510134359349 }, "model_routing": { "accuracy": 0.86, "cc": 72.32255475621157, "cc_by_type": { "choice": 88.74296435272043, "noul": 72.72727272727273, "score": 55.497427188641524 }, "low_n": false, "n": 150, "n_by_type": { "choice": 74, "noul": 44, "score": 32 }, "score": 72.32255475621157 }, "moderation_trust_safety": { "accuracy": 0.875, "cc": 69.1293608581969, "cc_by_type": { "choice": 99.99999999999999, "noul": 19.999999999999996, "score": 87.38808257459071 }, "low_n": true, "n": 16, "n_by_type": { "choice": 8, "noul": 5, "score": 3 }, "score": 69.1293608581969 }, "recruiting": { "accuracy": 0.6666666666666666, "cc": 38.80908171961838, "cc_by_type": { "choice": 69.84924623115576, "noul": 0.0, "score": 46.577998927699404 }, "low_n": true, "n": 21, "n_by_type": { "choice": 9, "noul": 6, "score": 6 }, "score": 38.80908171961838 }, "risk_assessment": { "accuracy": 0.38461538461538464, "cc": -15.99553483012555, "cc_by_type": { "choice": 43.39622641509434, "noul": -100.0, "score": 8.61716909452901 }, "low_n": true, "n": 13, "n_by_type": { "choice": 7, "noul": 3, "score": 3 }, "score": 0.0 }, "scientific_discovery": { "accuracy": 0.5625, "cc": 27.277877295181455, "cc_by_type": { "choice": 60.1593625498008, "noul": -14.28571428571429, "score": 35.959983621457866 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 27.277877295181455 }, "search_retrieval": { "accuracy": 0.9090909090909091, "cc": 84.20501734874512, "cc_by_type": { "choice": 85.68019093078759, "noul": 100.0, "score": 66.93486111544775 }, "low_n": true, "n": 22, "n_by_type": { "choice": 9, "noul": 6, "score": 7 }, "score": 84.20501734874512 } } }, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 84.77397078635877, 90.58056567199262 ], "I": [ 43.35399336962467, 54.02584879853301 ], "capability": [ 65.11784736173853, 71.70425623732149 ], "composite": { "A": [ 45.23103654293889, 64.78227929319183 ], "B": [ 41.83419561295226, 62.07842151813806 ], "C": [ 31.410442043707555, 52.48195783546016 ] }, "seed": 16 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": false, "g_med_used": 10.073244581339713, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 90.67498392881862, "capability": 69.9453997799703, "intelligence": 49.21581563112197 }, "per_type": { "choice": { "accuracy": 0.7693333333333333, "calibration": 90.35563030686254, "cc": 70.11573674209708, "cc_by_type": { "choice": 70.11573674209708 }, "cc_tiered": 71.21947685199589, "cc_tiered_open": 74.01883434266433, "cc_tiered_sealed": 68.42011936132744, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 70.11573674209708 }, "noul": { "accuracy": 0.616, "calibration": 90.17262756314747, "cc": 23.2, "cc_by_type": { "noul": 23.2 }, "cc_tiered": 24.522870023442106, "cc_tiered_open": 28.205659075224297, "cc_tiered_sealed": 20.840080971659916, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 23.2 }, "score": { "accuracy": 0.6293333333333333, "calibration": 91.49669391644588, "cc": 48.580321460438455, "cc_by_type": { "score": 48.580321460438455 }, "cc_tiered": 51.90510001792792, "cc_tiered_open": 57.66440539777861, "cc_tiered_sealed": 46.145794638077234, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 48.580321460438455 } }, "raw_on_A": { "calibration": 87.63270878627716, "capability": 67.620060964045, "intelligence": 47.60741314181283 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.014666666666666717, "n": 750 }, "noul": { "invalid_rate": 0.029333333333333322, "n": 375 }, "score": { "invalid_rate": 0.01866666666666672, "n": 375 } }, "views": { "Balanced 33:33:33": 55.71250293531964, "Emphasis on Accuracy 60:20:20": 51.855982453841044, "Emphasis on Cost 20:20:60": 52.04705676153498, "Emphasis on Speed 20:60:20": 65.14523654261257, "Intelligence only": 46.97811680206471 } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Thiago Silva / @devops_thiago (thi.winnt@gmail.com)", "axes": { "calibration": 48.5204641422835, "cost": 69.27998573050196, "intelligence": 28.182424563225457, "speed": 91.7664962211246 }, "calibration": { "parts": { "choice": { "ece_hard": 0.27998611987381716, "ece_score": 44.00277602523657, "mean_tvd": null, "n": 739, "n_ece_hard": 317, "n_tvd": 0, "score": 44.00277602523657 }, "noul": { "abstention_rate": 0.06593406593406592, "brier": 0.25247848434065967, "ece": 0.2512450549450551, "n": 364, "score": 49.750989010988974 }, "score": { "interval90_coverage": 0.7146739130434783, "interval90_width": 0.3179347826086956, "n": 368, "nrps": 0.19245016957880431, "score": 51.80762739062497, "top_ece": 0.38569864130434806 } }, "score": 48.5204641422835 }, "capability": 38.35144435275448, "composite_ci95": { "A": [ 8.912270303019026, 24.387188108020116 ], "B": [ 7.469929402481821, 21.841243683018 ], "C": [ 6.189076599318768, 16.935547297236194 ] }, "cost": { "basis": "ESTIMATE (*est.), exact same-base market reference; self-hosted open weights with no per-token tariff of their own. pricing_v15.reference() RAISES on every spelling of google/gemma-4-E4B-it - the base is in none of the three frozen tables - so this row carries its own LABELLED estimate from the reference the dated 25 Sep 2026 snapshot itself contains for exactly this base and size, deepinfra:google/gemma-4-E4B-it at USD 0.02 per 1M input and USD 0.1 per 1M output. The already-published classone-gemma4-e2b row publishes at that same rate as a one-size-UP estimate, so for this row it is the exact size rather than an upward approximation. RELEASE-LANE LINE, recorded and not acted on: the base needs either a listing decision or an UNLISTED_BASE_MODELS entry in the frozen table; no rate was invented here and the frozen module was not modified. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over the item's own candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone. This row's own usage on draw v1.6-regular-20261010-a2: 792,614 input and 0 output tokens over 1,500 decisions. Estimated, not charged.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.010568186666666667 }, "display": "ClassOne Gemma 4 E4B (Gemma-4-E4B backbone with ClassOne decision heads, schema-constrained single forward pass, self-hosted)", "endpoint_condition": "The author's own README Quickstart install line (pip install -e '.[dev]') at code pin github.com/devops-thiago/class-one@6cca132b and their own serve line (uvicorn classone.server.app:app); their server registers POST /v1/systemone itself, so the request and answer shapes are theirs. CLASSONE_BASE_MODEL was pointed at the LOCAL PINNED SNAPSHOT DIRECTORY, which their own code accepts, so the revision cannot drift to main and classone_heads.pt resolves locally with no request-time hub fetch. Their hard-coded MAX_TOKENS_PER_REQUEST = 8192 refuses longer prompts with HTTP 400; those items are recorded as invalid, the same way for every system. ONE documented mapping of ours and the only one: their server keys score probabilities '1'..'n' while JevBench's canonical keys are '0'..'n-1', so run 58's byte-identical ClassOneReindexAdapter relabels score keys k -> k-1 by calling the unchanged vendored typesafe adapter's own run(), and an item whose key set is not exactly {'1'..'n'} is REFUSED rather than rescored on a guess; choice and noul are untouched. The two already-published ClassOne rows carry exactly this treatment. evaluator-owned Lium RTX 5090 32 GB pod (pod calm-wolf-31, USD 0.72/h; Brasov, Romania, EU by the RIPE whois of its public IP, receipt r90/receipts/POD-IP-WHOIS-R90.txt); pod-level iptables egress cut installed, verified and TIMESTAMPED pod-side (logs/EGRESS-CUT-R90.log) BEFORE the sealed input was uploaded, so the ordering is provable from pod evidence and not only from the session transcript; HF/transformers offline, no credentials in the environment, one decision per request, no batching, no retry. The five rows ran strictly one at a time on the one card, each server started with setsid and stopped by its exact process group with a card-free check in between.", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 33.31401280028791, "I_sealed": 23.050836326163, "base": 28.182424563225457, "excess": 0.1899318927851983, "gap": 10.263176474124911, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 35.64521709768209, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 68.86486486486487, "hard": 20.729789241899976, "judge": 36.71720500988794, "standard": 47.25826705734617 } }, "open|noul": { "cc": 41.01932367149759, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 47.82608695652173, "judge": 10.000000000000009, "standard": 44.44444444444444 } }, "open|score": { "cc": 23.277497631684074, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 38.74636363636365, "hard": 25.742037037037047, "judge": 16.479661016949144, "standard": 20.81074074074073 } }, "sealed|choice": { "cc": 37.362851267778666, "n": { "easy": 72, "hard": 260, "judge": 141, "standard": 127 }, "tiers": { "easy": 38.054020231515274, "hard": 33.71441134439924, "judge": 36.076302572167826, "standard": 46.243969676085456 } }, "sealed|noul": { "cc": 31.78965771071034, "n": { "easy": 39, "hard": 132, "judge": 72, "standard": 57 }, "tiers": { "easy": 48.71794871794872, "hard": 10.606060606060597, "judge": 41.66666666666667, "standard": 50.877192982456144 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 33, "hard": 115, "judge": 75, "standard": 77 }, "tiers": { "easy": 14.18728971962614, "hard": -10.75497222222217, "judge": 1.1792035398229705, "standard": 4.336081081081044 } } } }, "jevbench_score": 15.607101626553092, "key": "classone-gemma4-e4b", "last_measured_on": "2026-10-10", "licence": "apache-2.0", "listing": "ranked", "measured_in": "v1.6.9", "measurement_date_status": "measured", "model_pin": "devops-thiago/classone-gemma4-e4b@74d6e08152878a4956ce9ffc323f6c06f13ca9af", "noul_decisive": { "acc_among_decisive": 0.7382352941176471, "decisive": 340, "decisive_rate": 0.9340659340659341, "n": 375, "supported": true, "valid": 364 }, "planned_weight_denominator": 1.0, "rank": 6, "ranked": true, "ranks": { "A": 6, "B": 6, "C": 6, "capability": 10 }, "repo": "https://huggingface.co/devops-thiago/classone-gemma4-e4b", "scores": { "A": 15.607101626553092, "B": 13.587651382976851, "C": 10.838265018439648 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.22830128003843128, "p50_s_raw": 0.03915064001921564, "p95_s_adjusted": 0.29163660332560537, "p95_s_raw": 0.07081830166280269 }, "status": { "answered_ok": 1471, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns": { "family": { "calibration": { "accuracy": 0.6551724137931034, "cc": 48.403611111111104, "cc_by_type": { "choice": 66.66666666666667, "noul": 66.66666666666667, "score": 11.877499999999985 }, "low_n": true, "n": 29, "n_by_type": { "choice": 16, "noul": 6, "score": 7 }, "score": 48.403611111111104 }, "calibration_gen": { "accuracy": 0.42857142857142855, "cc": 2.0163035515209473, "cc_by_type": { "choice": 32.453416149068325, "noul": 23.076923076923084, "score": -49.481428571428566 }, "low_n": false, "n": 112, "n_by_type": { "choice": 56, "noul": 26, "score": 30 }, "score": 2.0163035515209473 }, "composite_gen": { "accuracy": 0.5102040816326531, "cc": 24.397270723104047, "cc_by_type": { "choice": 40.74074074074074, "noul": 21.42857142857142, "score": 11.02249999999999 }, "low_n": false, "n": 98, "n_by_type": { "choice": 36, "noul": 28, "score": 34 }, "score": 24.397270723104047 }, "fairness": { "accuracy": 0.7777777777777778, "cc": 78.00062745098039, "cc_by_type": { "choice": 64.70588235294117, "noul": 100.0, "score": 69.296 }, "low_n": true, "n": 27, "n_by_type": { "choice": 12, "noul": 6, "score": 9 }, "score": 78.00062745098039 }, "jaggedness": { "accuracy": 0.3837837837837838, "cc": 11.209656087483806, "cc_by_type": { "choice": 18.017600741083836, "noul": 23.076923076923084, "score": -7.465555555555503 }, "low_n": false, "n": 185, "n_by_type": { "choice": 97, "noul": 39, "score": 49 }, "score": 11.209656087483806 }, "llm_router": { "accuracy": 0.6363636363636364, "cc": 55.196760619175414, "cc_by_type": { "choice": 47.05062703204832, "noul": 61.64383561643836, "score": 56.895819209039544 }, "low_n": false, "n": 264, "n_by_type": { "choice": 131, "noul": 73, "score": 60 }, "score": 55.196760619175414 }, "long_state": { "accuracy": 0.0, "cc": -110.31746031746032, "cc_by_type": { "choice": -30.952380952380945, "noul": -100.0, "score": -200.0 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 11, "score": 7 }, "score": 0.0 }, "ml_usecase": { "accuracy": 0.5161290322580645, "cc": 23.0149386845039, "cc_by_type": { "choice": 16.923076923076923, "noul": 50.0, "score": 2.1217391304347744 }, "low_n": false, "n": 31, "n_by_type": { "choice": 15, "noul": 8, "score": 8 }, "score": 23.0149386845039 }, "multilingual": { "accuracy": 0.5365079365079365, "cc": 29.473782295572374, "cc_by_type": { "choice": 43.573667711598766, "noul": 40.22988505747127, "score": 4.617794117647089 }, "low_n": false, "n": 315, "n_by_type": { "choice": 146, "noul": 87, "score": 82 }, "score": 29.473782295572374 }, "negation_pair_gen": { "accuracy": 0.48214285714285715, "cc": -16.955793650793655, "cc_by_type": { "choice": 19.047619047619047, "noul": 30.000000000000004, "score": -99.91500000000002 }, "low_n": false, "n": 56, "n_by_type": { "choice": 28, "noul": 20, "score": 8 }, "score": 0.0 }, "option_order": { "accuracy": 0.38596491228070173, "cc": 22.30854605993341, "cc_by_type": { "choice": 22.30854605993341 }, "low_n": false, "n": 57, "n_by_type": { "choice": 57 }, "score": 22.30854605993341 }, "perturb_pair": { "accuracy": 0.75, "cc": 67.99352945343772, "cc_by_type": { "choice": 64.61336828309307, "noul": 71.42857142857142, "score": 67.93864864864865 }, "low_n": false, "n": 60, "n_by_type": { "choice": 34, "noul": 14, "score": 12 }, "score": 67.99352945343772 }, "usecase": { "accuracy": 0.46835443037974683, "cc": 14.865840833856993, "cc_by_type": { "choice": 42.18539217575643, "noul": 12.280701754385959, "score": -9.868571428571403 }, "low_n": false, "n": 237, "n_by_type": { "choice": 111, "noul": 57, "score": 69 }, "score": 14.865840833856993 } }, "lang": { "ar": { "accuracy": 0.6363636363636364, "cc": 26.925555555555565, "cc_by_type": { "choice": 46.666666666666664, "noul": 50.0, "score": -15.88999999999996 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 4, "score": 2 }, "score": 26.925555555555565 }, "cs": { "accuracy": 0.5, "cc": 7.90960451977401, "cc_by_type": { "choice": 49.152542372881356, "noul": -33.333333333333336 }, "low_n": true, "n": 8, "n_by_type": { "choice": 5, "noul": 3 }, "score": 7.90960451977401 }, "da": { "accuracy": 0.4166666666666667, "cc": 24.536203155818544, "cc_by_type": { "choice": 17.159763313609464, "noul": 60.00000000000001, "score": -3.5511538461538317 }, "low_n": true, "n": 24, "n_by_type": { "choice": 11, "noul": 5, "score": 8 }, "score": 24.536203155818544 }, "de": { "accuracy": 0.5333333333333333, "cc": 21.35741575994569, "cc_by_type": { "choice": 56.54797827398914, "noul": 11.111111111111116, "score": -3.5868421052631883 }, "low_n": false, "n": 45, "n_by_type": { "choice": 24, "noul": 9, "score": 12 }, "score": 21.35741575994569 }, "el": { "accuracy": 0.4444444444444444, "cc": 0.07861445783131475, "cc_by_type": { "choice": 27.710843373493972, "noul": -19.999999999999996, "score": -7.475000000000032 }, "low_n": true, "n": 18, "n_by_type": { "choice": 11, "noul": 5, "score": 2 }, "score": 0.07861445783131475 }, "en": { "accuracy": 0.49508489722966936, "cc": 22.359010078249664, "cc_by_type": { "choice": 35.3897924845765, "noul": 28.57142857142858, "score": 3.1158091787439113 }, "low_n": false, "n": 1119, "n_by_type": { "choice": 577, "noul": 266, "score": 276 }, "score": 22.359010078249664 }, "es": { "accuracy": 0.7857142857142857, "cc": 58.303939393939395, "cc_by_type": { "choice": 73.81818181818181, "noul": 100.0, "score": 1.0936363636363633 }, "low_n": true, "n": 28, "n_by_type": { "choice": 10, "noul": 11, "score": 7 }, "score": 58.303939393939395 }, "fi": { "accuracy": 0.4444444444444444, "cc": 47.183098591549296, "cc_by_type": { "choice": -5.63380281690141, "noul": 100.0 }, "low_n": true, "n": 9, "n_by_type": { "choice": 6, "noul": 3 }, "score": 47.183098591549296 }, "fr": { "accuracy": 0.6, "cc": 30.50427698300759, "cc_by_type": { "choice": 49.22425952045134, "noul": 0.0, "score": 42.28857142857142 }, "low_n": true, "n": 15, "n_by_type": { "choice": 5, "noul": 4, "score": 6 }, "score": 30.50427698300759 }, "hi": { "accuracy": 0.5, "cc": 33.68955033643892, "cc_by_type": { "choice": 25.465838509316765, "noul": 60.00000000000001, "score": 15.602812499999995 }, "low_n": false, "n": 30, "n_by_type": { "choice": 11, "noul": 10, "score": 9 }, "score": 33.68955033643892 }, "id": { "accuracy": 0.5, "cc": 17.741935483870968, "cc_by_type": { "choice": 35.483870967741936, "noul": 0.0 }, "low_n": true, "n": 6, "n_by_type": { "choice": 4, "noul": 2 }, "score": 17.741935483870968 }, "it": { "accuracy": 0.7058823529411765, "cc": 57.02479166666668, "cc_by_type": { "choice": 100.00000000000001, "noul": 50.0, "score": 21.07437500000001 }, "low_n": true, "n": 17, "n_by_type": { "choice": 5, "noul": 4, "score": 8 }, "score": 57.02479166666668 }, "ja": { "accuracy": 0.375, "cc": -0.5040061217576456, "cc_by_type": { "choice": 18.69918699186992, "noul": -42.85714285714286, "score": 22.645937500000002 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 0.0 }, "ko": { "accuracy": 0.7272727272727273, "cc": 77.51933333333334, "cc_by_type": { "choice": 100.0, "noul": 100.0, "score": 32.55800000000002 }, "low_n": true, "n": 11, "n_by_type": { "choice": 4, "noul": 2, "score": 5 }, "score": 77.51933333333334 }, "mixed": { "accuracy": 1.0, "cc": 98.75666666666667, "cc_by_type": { "choice": 100.0, "noul": 100.0, "score": 96.27000000000004 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 4, "score": 1 }, "score": 98.75666666666667 }, "nl": { "accuracy": 0.38461538461538464, "cc": -15.442971216341691, "cc_by_type": { "choice": 24.79108635097493, "noul": 0.0, "score": -71.12 }, "low_n": true, "n": 13, "n_by_type": { "choice": 5, "noul": 6, "score": 2 }, "score": 0.0 }, "no": { "accuracy": 0.42857142857142855, "cc": 34.42622950819672, "cc_by_type": { "choice": -31.14754098360656, "noul": 100.0 }, "low_n": true, "n": 7, "n_by_type": { "choice": 4, "noul": 3 }, "score": 34.42622950819672 }, "pl": { "accuracy": 0.625, "cc": 50.14468360745798, "cc_by_type": { "choice": 76.38828159160472, "noul": 80.0, "score": -5.954230769230784 }, "low_n": false, "n": 40, "n_by_type": { "choice": 16, "noul": 10, "score": 14 }, "score": 50.14468360745798 }, "pt": { "accuracy": 0.6111111111111112, "cc": 49.75193662758235, "cc_by_type": { "choice": 45.7286432160804, "noul": 66.66666666666667, "score": 36.86049999999999 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 6, "score": 5 }, "score": 49.75193662758235 }, "sv": { "accuracy": 0.5, "cc": -7.282962962962962, "cc_by_type": { "choice": 11.111111111111109, "noul": 50.0, "score": -82.96 }, "low_n": true, "n": 8, "n_by_type": { "choice": 3, "noul": 4, "score": 1 }, "score": 0.0 }, "tr": { "accuracy": 0.7272727272727273, "cc": 23.552514619883052, "cc_by_type": { "choice": 73.6842105263158, "noul": 100.0, "score": -103.02666666666664 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 3, "score": 3 }, "score": 23.552514619883052 }, "uk": { "accuracy": 0.2727272727272727, "cc": -34.453968784838345, "cc_by_type": { "choice": 13.043478260869565, "noul": -100.0, "score": -16.405384615384612 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 1, "score": 4 }, "score": 0.0 }, "zh": { "accuracy": 0.625, "cc": 36.97171171171172, "cc_by_type": { "choice": 35.13513513513514, "noul": 100.0, "score": -24.219999999999974 }, "low_n": true, "n": 8, "n_by_type": { "choice": 4, "noul": 3, "score": 1 }, "score": 36.97171171171172 } }, "type": { "choice": { "accuracy": 0.5173333333333333, "cc": 37.46761098635343, "cc_by_type": { "choice": 37.46761098635343 }, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 37.46761098635343 }, "noul": { "accuracy": 0.6693333333333333, "cc": 33.86666666666667, "cc_by_type": { "noul": 33.86666666666667 }, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 33.86666666666667 }, "score": { "accuracy": 0.3413333333333333, "cc": 3.9127145359018956, "cc_by_type": { "score": 3.9127145359018956 }, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 3.9127145359018956 } }, "usecase": { "advertising": { "accuracy": 0.5, "cc": -21.09269082921473, "cc_by_type": { "choice": 40.691927512355846, "noul": 19.999999999999996, "score": -123.97000000000004 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 0.0 }, "code_linting": { "accuracy": 0.3333333333333333, "cc": 6.366578355155487, "cc_by_type": { "choice": -8.01963993453355, "noul": 60.00000000000001, "score": -32.880624999999995 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 6.366578355155487 }, "customer_support": { "accuracy": 0.6, "cc": 34.63327543424317, "cc_by_type": { "choice": 51.61290322580645, "noul": 25.0, "score": 27.286923076923063 }, "low_n": true, "n": 20, "n_by_type": { "choice": 8, "noul": 8, "score": 4 }, "score": 34.63327543424317 }, "demand_forecasting": { "accuracy": 0.23076923076923078, "cc": -1.8498271875380354, "cc_by_type": { "choice": 3.614457831325295, "noul": 0.0, "score": -9.163939393939401 }, "low_n": true, "n": 26, "n_by_type": { "choice": 11, "noul": 4, "score": 11 }, "score": 0.0 }, "ecommerce_marketplaces": { "accuracy": 0.2777777777777778, "cc": -8.013566650740566, "cc_by_type": { "choice": 25.465838509316768, "noul": -60.0, "score": 10.493461538461535 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 5, "score": 6 }, "score": 0.0 }, "feature_extraction": { "accuracy": 0.5172413793103449, "cc": 16.519821289821277, "cc_by_type": { "choice": 48.71794871794871, "noul": 19.999999999999996, "score": -19.158484848484882 }, "low_n": true, "n": 29, "n_by_type": { "choice": 15, "noul": 5, "score": 9 }, "score": 16.519821289821277 }, "financial_crime": { "accuracy": 0.5384615384615384, "cc": 24.41154782608697, "cc_by_type": { "choice": 73.91304347826087, "noul": 25.0, "score": -25.678399999999968 }, "low_n": true, "n": 26, "n_by_type": { "choice": 10, "noul": 8, "score": 8 }, "score": 24.41154782608697 }, "gaming": { "accuracy": 0.3793103448275862, "cc": 6.763275341819888, "cc_by_type": { "choice": 16.831683168316832, "noul": 42.85714285714286, "score": -39.39900000000003 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 7, "score": 11 }, "score": 6.763275341819888 }, "insurance_claims": { "accuracy": 0.2857142857142857, "cc": -46.41502645502644, "cc_by_type": { "choice": 20.634920634920636, "noul": -60.0, "score": -99.87999999999995 }, "low_n": true, "n": 14, "n_by_type": { "choice": 8, "noul": 5, "score": 1 }, "score": 0.0 }, "knowledge_graphs": { "accuracy": 0.21739130434782608, "cc": -6.9660931689176, "cc_by_type": { "choice": -9.923664122137408, "noul": 19.999999999999996, "score": -30.97461538461539 }, "low_n": true, "n": 23, "n_by_type": { "choice": 14, "noul": 5, "score": 4 }, "score": 0.0 }, "lead_generation": { "accuracy": 0.4, "cc": 7.34117073170732, "cc_by_type": { "choice": 51.21951219512196, "noul": 0.0, "score": -29.195999999999998 }, "low_n": true, "n": 25, "n_by_type": { "choice": 8, "noul": 8, "score": 9 }, "score": 7.34117073170732 }, "legal_compliance": { "accuracy": 0.5384615384615384, "cc": 5.378939247830284, "cc_by_type": { "choice": 34.42622950819673, "noul": -100.0, "score": 81.71058823529413 }, "low_n": true, "n": 13, "n_by_type": { "choice": 8, "noul": 1, "score": 4 }, "score": 5.378939247830284 }, "llm_guardrails": { "accuracy": 0.6551724137931034, "cc": 45.51440602211244, "cc_by_type": { "choice": 72.4770642201835, "noul": 50.0, "score": 14.066153846153838 }, "low_n": true, "n": 29, "n_by_type": { "choice": 14, "noul": 8, "score": 7 }, "score": 45.51440602211244 }, "model_routing": { "accuracy": 0.7466666666666667, "cc": 66.53999293560527, "cc_by_type": { "choice": 67.83704100777273, "noul": 77.27272727272727, "score": 54.510210526315795 }, "low_n": false, "n": 150, "n_by_type": { "choice": 74, "noul": 44, "score": 32 }, "score": 66.53999293560527 }, "moderation_trust_safety": { "accuracy": 0.5625, "cc": 31.290701754385953, "cc_by_type": { "choice": 36.84210526315789, "noul": 60.00000000000001, "score": -2.970000000000028 }, "low_n": true, "n": 16, "n_by_type": { "choice": 8, "noul": 5, "score": 3 }, "score": 31.290701754385953 }, "recruiting": { "accuracy": 0.6666666666666666, "cc": 37.53701370098354, "cc_by_type": { "choice": 69.84924623115576, "noul": 33.33333333333333, "score": 9.42846153846154 }, "low_n": true, "n": 21, "n_by_type": { "choice": 9, "noul": 6, "score": 6 }, "score": 37.53701370098354 }, "risk_assessment": { "accuracy": 0.6153846153846154, "cc": 37.185966981132076, "cc_by_type": { "choice": 62.264150943396224, "noul": 100.0, "score": -50.70625 }, "low_n": true, "n": 13, "n_by_type": { "choice": 7, "noul": 3, "score": 3 }, "score": 37.185966981132076 }, "scientific_discovery": { "accuracy": 0.28125, "cc": 2.9136300828432273, "cc_by_type": { "choice": -3.585657370517929, "noul": 14.28571428571428, "score": -1.9591666666666674 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 2.9136300828432273 }, "search_retrieval": { "accuracy": 0.6818181818181818, "cc": 46.84529194088859, "cc_by_type": { "choice": 57.04057279236276, "noul": 66.66666666666667, "score": 16.828636363636363 }, "low_n": true, "n": 22, "n_by_type": { "choice": 9, "noul": 6, "score": 7 }, "score": 46.84529194088859 } } }, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 42.95152007629359, 53.0806033344027 ], "I": [ 22.568457707555886, 33.716071635910566 ], "capability": [ 33.46902235931629, 42.57940480198994 ], "composite": { "A": [ 8.912270303019026, 24.387188108020116 ], "B": [ 7.469929402481821, 21.841243683018 ], "C": [ 6.189076599318768, 16.935547297236194 ] }, "seed": 17 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": false, "g_med_used": 10.073244581339713, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 48.5204641422835, "capability": 38.35144435275448, "intelligence": 28.182424563225457 }, "per_type": { "choice": { "accuracy": 0.5173333333333333, "calibration": 44.00277602523657, "cc": 37.46761098635343, "cc_by_type": { "choice": 37.46761098635343 }, "cc_tiered": 36.50403418273038, "cc_tiered_open": 35.64521709768209, "cc_tiered_sealed": 37.362851267778666, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 37.46761098635343 }, "noul": { "accuracy": 0.6693333333333333, "calibration": 49.750989010988974, "cc": 33.86666666666667, "cc_by_type": { "noul": 33.86666666666667 }, "cc_tiered": 36.404490691103966, "cc_tiered_open": 41.01932367149759, "cc_tiered_sealed": 31.78965771071034, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 33.86666666666667 }, "score": { "accuracy": 0.3413333333333333, "calibration": 51.80762739062497, "cc": 3.9127145359018956, "cc_by_type": { "score": 3.9127145359018956 }, "cc_tiered": 11.638748815842037, "cc_tiered_open": 23.277497631684074, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 3.9127145359018956 } }, "raw_on_A": { "calibration": 43.42824044580669, "capability": 35.61702293170774, "intelligence": 27.805805417608788 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.014666666666666717, "n": 750 }, "noul": { "invalid_rate": 0.029333333333333322, "n": 375 }, "score": { "invalid_rate": 0.01866666666666672, "n": 375 } }, "views": { "Balanced 33:33:33": 15.672226976099346, "Emphasis on Accuracy 60:20:20": 12.054100187818078, "Emphasis on Cost 20:20:60": 17.712389550075887, "Emphasis on Speed 20:60:20": 19.22912598635703, "Intelligence only": 8.95354562245959 } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Thiago Silva / @devops_thiago (thi.winnt@gmail.com)", "axes": { "calibration": 32.62324971930184, "cost": 70.24966415777286, "intelligence": 5.814527660927306, "speed": 93.11279379161611 }, "calibration": { "parts": { "choice": { "ece_hard": 0.42988044164037864, "ece_score": 14.023911671924271, "mean_tvd": null, "n": 739, "n_ece_hard": 317, "n_tvd": 0, "score": 14.023911671924271 }, "noul": { "abstention_rate": 0.12637362637362637, "brier": 0.353035391263736, "ece": 0.33436318681318694, "n": 364, "score": 33.12736263736261 }, "score": { "interval90_coverage": 0.7309782608695652, "interval90_width": 0.45244565217391297, "n": 368, "nrps": 0.215767459549366, "score": 50.718474848618655, "top_ece": 0.3849315217391305 } }, "score": 32.62324971930184 }, "capability": 19.21888869011457, "composite_ci95": { "A": [ 0.07421330468385162, 1.5542613561325538 ], "B": [ 0.051005668190310965, 1.2102758221999665 ], "C": [ 0.051537017141563626, 1.0793481639809404 ] }, "cost": { "basis": "ESTIMATE (*est.), exact base-model market reference; self-hosted open weights with no per-token tariff of their own. The unmodified frozen pricing_v15 resolves the author's own declared base spelling Qwen/Qwen3.5-2B straight through to deepinfra:Qwen/Qwen3.5-2B at USD 0.02 per 1M input and USD 0.1 per 1M output in the dated 25 Sep 2026 snapshot. No new release-lane pricing line. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over the item's own candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone. This row's own usage on draw v1.6-regular-20261010-a2: 735,765 input and 0 output tokens over 1,500 decisions. Estimated, not charged.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.009810200000000002 }, "display": "ClassOne Qwen 3.5 2B (Qwen3.5-2B backbone with ClassOne decision heads, schema-constrained single forward pass, self-hosted)", "endpoint_condition": "The author's own README Quickstart install line (pip install -e '.[dev]') at code pin github.com/devops-thiago/class-one@6cca132b and their own serve line (uvicorn classone.server.app:app); their server registers POST /v1/systemone itself, so the request and answer shapes are theirs. CLASSONE_BASE_MODEL was pointed at the LOCAL PINNED SNAPSHOT DIRECTORY, which their own code accepts, so the revision cannot drift to main and classone_heads.pt resolves locally with no request-time hub fetch. Their hard-coded MAX_TOKENS_PER_REQUEST = 8192 refuses longer prompts with HTTP 400; those items are recorded as invalid, the same way for every system. ONE documented mapping of ours and the only one: their server keys score probabilities '1'..'n' while JevBench's canonical keys are '0'..'n-1', so run 58's byte-identical ClassOneReindexAdapter relabels score keys k -> k-1 by calling the unchanged vendored typesafe adapter's own run(), and an item whose key set is not exactly {'1'..'n'} is REFUSED rather than rescored on a guess; choice and noul are untouched. The two already-published ClassOne rows carry exactly this treatment. evaluator-owned Lium RTX 5090 32 GB pod (pod calm-wolf-31, USD 0.72/h; Brasov, Romania, EU by the RIPE whois of its public IP, receipt r90/receipts/POD-IP-WHOIS-R90.txt); pod-level iptables egress cut installed, verified and TIMESTAMPED pod-side (logs/EGRESS-CUT-R90.log) BEFORE the sealed input was uploaded, so the ordering is provable from pod evidence and not only from the session transcript; HF/transformers offline, no credentials in the environment, one decision per request, no batching, no retry. The five rows ran strictly one at a time on the one card, each server started with setsid and stopped by its exact process group with a card-free check in between.", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 6.892803202604126, "I_sealed": 4.736252119250485, "base": 5.814527660927306, "excess": -7.916693497986072, "gap": 2.1565510833536408, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 18.56541234955058, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 53.297297297297305, "hard": 16.955017301038072, "judge": 5.075807514831907, "standard": 24.65466722478024 } }, "open|noul": { "cc": 1.9710144927536213, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": -21.739130434782606, "judge": -19.999999999999996, "standard": 33.33333333333333 } }, "open|score": { "cc": 0.14198276550817734, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 50.27727272727272, "hard": -2.0812037037037046, "judge": 4.7081355932203195, "standard": -27.328518518518543 } }, "sealed|choice": { "cc": 10.956701395170182, "n": { "easy": 72, "hard": 260, "judge": 141, "standard": 127 }, "tiers": { "easy": 19.282511210762333, "hard": 9.970319885676586, "judge": 3.2012581807112888, "standard": 20.399724328049636 } }, "sealed|noul": { "cc": 3.2520549625812754, "n": { "easy": 39, "hard": 132, "judge": 72, "standard": 57 }, "tiers": { "easy": 38.46153846153846, "hard": -12.121212121212121, "judge": 8.333333333333325, "standard": 8.771929824561408 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 33, "hard": 115, "judge": 75, "standard": 77 }, "tiers": { "easy": 16.134112149532708, "hard": -13.1237222222222, "judge": -24.52305309734517, "standard": -9.89180180180178 } } } }, "jevbench_score": 0.23766015025378578, "key": "classone-qwen3.5-2b", "last_measured_on": "2026-10-10", "licence": "apache-2.0", "listing": "ranked", "measured_in": "v1.6.9", "measurement_date_status": "measured", "model_pin": "devops-thiago/classone-qwen3.5-2b@fc15401b6ae6d86b8fdf664b40bfdd8055edf374", "noul_decisive": { "acc_among_decisive": 0.6226415094339622, "decisive": 318, "decisive_rate": 0.8736263736263736, "n": 375, "supported": true, "valid": 364 }, "planned_weight_denominator": 1.0, "rank": 13, "ranked": true, "ranks": { "A": 13, "B": 13, "C": 13, "capability": 16 }, "repo": "https://huggingface.co/devops-thiago/classone-qwen3.5-2b", "scores": { "A": 0.23766015025378578, "B": 0.1692155731407913, "C": 0.16504177100957346 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.21041937693953514, "p50_s_raw": 0.03020968846976757, "p95_s_adjusted": 0.23207849023165178, "p95_s_raw": 0.0410392451158259 }, "status": { "answered_ok": 1471, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns": { "family": { "calibration": { "accuracy": 0.4827586206896552, "cc": 26.97833333333334, "cc_by_type": { "choice": 8.333333333333334, "noul": 66.66666666666667, "score": 5.935000000000013 }, "low_n": true, "n": 29, "n_by_type": { "choice": 16, "noul": 6, "score": 7 }, "score": 26.97833333333334 }, "calibration_gen": { "accuracy": 0.3392857142857143, "cc": -13.726849020544668, "cc_by_type": { "choice": 16.14906832298137, "noul": -15.384615384615385, "score": -41.944999999999986 }, "low_n": false, "n": 112, "n_by_type": { "choice": 56, "noul": 26, "score": 30 }, "score": 0.0 }, "composite_gen": { "accuracy": 0.32653061224489793, "cc": 0.7510267857142816, "cc_by_type": { "choice": 0.0, "noul": -7.14285714285714, "score": 9.395937499999985 }, "low_n": false, "n": 98, "n_by_type": { "choice": 36, "noul": 28, "score": 34 }, "score": 0.7510267857142816 }, "fairness": { "accuracy": 0.4444444444444444, "cc": -7.00355555555556, "cc_by_type": { "choice": 100.0, "noul": -100.0, "score": -21.010666666666687 }, "low_n": true, "n": 27, "n_by_type": { "choice": 12, "noul": 6, "score": 9 }, "score": 0.0 }, "jaggedness": { "accuracy": 0.3837837837837838, "cc": 5.861599107426817, "cc_by_type": { "choice": 18.017600741083836, "noul": 7.692307692307687, "score": -8.125111111111071 }, "low_n": false, "n": 185, "n_by_type": { "choice": 97, "noul": 39, "score": 49 }, "score": 5.861599107426817 }, "llm_router": { "accuracy": 0.48863636363636365, "cc": 33.92782003547969, "cc_by_type": { "choice": 21.96934509986067, "noul": 42.46575342465753, "score": 37.34836158192088 }, "low_n": false, "n": 264, "n_by_type": { "choice": 131, "noul": 73, "score": 60 }, "score": 33.92782003547969 }, "long_state": { "accuracy": 0.0, "cc": -110.31746031746032, "cc_by_type": { "choice": -30.952380952380945, "noul": -100.0, "score": -200.0 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 11, "score": 7 }, "score": 0.0 }, "ml_usecase": { "accuracy": 0.3225806451612903, "cc": -20.34245261984394, "cc_by_type": { "choice": 16.923076923076923, "noul": -25.0, "score": -52.950434782608745 }, "low_n": false, "n": 31, "n_by_type": { "choice": 15, "noul": 8, "score": 8 }, "score": 0.0 }, "multilingual": { "accuracy": 0.38412698412698415, "cc": 4.092899567611803, "cc_by_type": { "choice": 13.121361397223472, "noul": 8.045977011494255, "score": -8.888639705882317 }, "low_n": false, "n": 315, "n_by_type": { "choice": 146, "noul": 87, "score": 82 }, "score": 4.092899567611803 }, "negation_pair_gen": { "accuracy": 0.39285714285714285, "cc": -26.347896825396816, "cc_by_type": { "choice": 9.523809523809527, "noul": 10.000000000000009, "score": -98.56749999999998 }, "low_n": false, "n": 56, "n_by_type": { "choice": 28, "noul": 20, "score": 8 }, "score": 0.0 }, "option_order": { "accuracy": 0.21052631578947367, "cc": 0.11098779134295962, "cc_by_type": { "choice": 0.11098779134295962 }, "low_n": false, "n": 57, "n_by_type": { "choice": 57 }, "score": 0.11098779134295962 }, "perturb_pair": { "accuracy": 0.5666666666666667, "cc": 48.77980199071942, "cc_by_type": { "choice": 25.29488859764089, "noul": 71.42857142857142, "score": 49.61594594594595 }, "low_n": false, "n": 60, "n_by_type": { "choice": 34, "noul": 14, "score": 12 }, "score": 48.77980199071942 }, "usecase": { "accuracy": 0.3206751054852321, "cc": -11.511251481717963, "cc_by_type": { "choice": 6.340335324725405, "noul": -19.298245614035093, "score": -21.575844155844194 }, "low_n": false, "n": 237, "n_by_type": { "choice": 111, "noul": 57, "score": 69 }, "score": 0.0 } }, "lang": { "ar": { "accuracy": 0.45454545454545453, "cc": -20.115, "cc_by_type": { "choice": 20.000000000000004, "noul": 0.0, "score": -80.345 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 4, "score": 2 }, "score": 0.0 }, "cs": { "accuracy": 0.5, "cc": 49.152542372881356, "cc_by_type": { "choice": -1.694915254237285, "noul": 100.0 }, "low_n": true, "n": 8, "n_by_type": { "choice": 5, "noul": 3 }, "score": 49.152542372881356 }, "da": { "accuracy": 0.375, "cc": -13.013510848126225, "cc_by_type": { "choice": 28.994082840236683, "noul": -60.0, "score": -8.034615384615362 }, "low_n": true, "n": 24, "n_by_type": { "choice": 11, "noul": 5, "score": 8 }, "score": 0.0 }, "de": { "accuracy": 0.28888888888888886, "cc": -16.378460738952608, "cc_by_type": { "choice": 2.232951116475559, "noul": -33.333333333333336, "score": -18.035000000000046 }, "low_n": false, "n": 45, "n_by_type": { "choice": 24, "noul": 9, "score": 12 }, "score": 0.0 }, "el": { "accuracy": 0.2222222222222222, "cc": -34.461847389558244, "cc_by_type": { "choice": 3.614457831325295, "noul": -60.0, "score": -47.00000000000002 }, "low_n": true, "n": 18, "n_by_type": { "choice": 11, "noul": 5, "score": 2 }, "score": 0.0 }, "en": { "accuracy": 0.3708668453976765, "cc": 2.919018796085924, "cc_by_type": { "choice": 12.955692652832235, "noul": 2.2556390977443552, "score": -6.454275362318818 }, "low_n": false, "n": 1119, "n_by_type": { "choice": 577, "noul": 266, "score": 276 }, "score": 2.919018796085924 }, "es": { "accuracy": 0.42857142857142855, "cc": -0.37590909090909125, "cc_by_type": { "choice": 47.63636363636363, "noul": -27.27272727272727, "score": -21.491363636363637 }, "low_n": true, "n": 28, "n_by_type": { "choice": 10, "noul": 11, "score": 7 }, "score": 0.0 }, "fi": { "accuracy": 0.4444444444444444, "cc": 47.183098591549296, "cc_by_type": { "choice": -5.63380281690141, "noul": 100.0 }, "low_n": true, "n": 9, "n_by_type": { "choice": 6, "noul": 3 }, "score": 47.183098591549296 }, "fr": { "accuracy": 0.4, "cc": -2.847552779456862, "cc_by_type": { "choice": 23.83638928067701, "noul": 0.0, "score": -32.3790476190476 }, "low_n": true, "n": 15, "n_by_type": { "choice": 5, "noul": 4, "score": 6 }, "score": 0.0 }, "hi": { "accuracy": 0.4, "cc": 11.143196169772247, "cc_by_type": { "choice": 25.465838509316765, "noul": 39.99999999999999, "score": -32.03625000000001 }, "low_n": false, "n": 30, "n_by_type": { "choice": 11, "noul": 10, "score": 9 }, "score": 11.143196169772247 }, "id": { "accuracy": 0.3333333333333333, "cc": 35.483870967741936, "cc_by_type": { "choice": -29.032258064516125, "noul": 100.0 }, "low_n": true, "n": 6, "n_by_type": { "choice": 4, "noul": 2 }, "score": 35.483870967741936 }, "it": { "accuracy": 0.5882352941176471, "cc": 39.10954484015044, "cc_by_type": { "choice": 49.22425952045134, "noul": 50.0, "score": 18.104374999999983 }, "low_n": true, "n": 17, "n_by_type": { "choice": 5, "noul": 4, "score": 8 }, "score": 39.10954484015044 }, "ja": { "accuracy": 0.28125, "cc": -2.907849641889275, "cc_by_type": { "choice": -13.821138211382111, "noul": 14.28571428571428, "score": -9.188124999999992 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 0.0 }, "ko": { "accuracy": 0.36363636363636365, "cc": 15.881704918032781, "cc_by_type": { "choice": 67.21311475409836, "noul": 0.0, "score": -19.56800000000001 }, "low_n": true, "n": 11, "n_by_type": { "choice": 4, "noul": 2, "score": 5 }, "score": 15.881704918032781 }, "mixed": { "accuracy": 0.7272727272727273, "cc": 58.55131868131869, "cc_by_type": { "choice": 56.04395604395604, "noul": 50.0, "score": 69.61000000000004 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 4, "score": 1 }, "score": 58.55131868131869 }, "nl": { "accuracy": 0.38461538461538464, "cc": 5.939593933766626, "cc_by_type": { "choice": -0.2785515320334245, "noul": 33.33333333333333, "score": -15.236000000000027 }, "low_n": true, "n": 13, "n_by_type": { "choice": 5, "noul": 6, "score": 2 }, "score": 5.939593933766626 }, "no": { "accuracy": 0.7142857142857143, "cc": 67.21311475409837, "cc_by_type": { "choice": 34.42622950819673, "noul": 100.0 }, "low_n": true, "n": 7, "n_by_type": { "choice": 4, "noul": 3 }, "score": 67.21311475409837 }, "pl": { "accuracy": 0.475, "cc": 18.48718910116263, "cc_by_type": { "choice": 44.90599038041102, "noul": 39.99999999999999, "score": -29.444423076923123 }, "low_n": false, "n": 40, "n_by_type": { "choice": 16, "noul": 10, "score": 14 }, "score": 18.48718910116263 }, "pt": { "accuracy": 0.5, "cc": 18.251714405360136, "cc_by_type": { "choice": 45.7286432160804, "noul": 0.0, "score": 9.026500000000004 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 6, "score": 5 }, "score": 18.251714405360136 }, "sv": { "accuracy": 0.25, "cc": -12.537777777777777, "cc_by_type": { "choice": -33.333333333333336, "noul": 0.0, "score": -4.279999999999995 }, "low_n": true, "n": 8, "n_by_type": { "choice": 3, "noul": 4, "score": 1 }, "score": 0.0 }, "tr": { "accuracy": 0.45454545454545453, "cc": 0.6930994152046814, "cc_by_type": { "choice": 21.052631578947373, "noul": -33.333333333333336, "score": 14.360000000000007 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 3, "score": 3 }, "score": 0.6930994152046814 }, "uk": { "accuracy": 0.45454545454545453, "cc": -18.83422519509476, "cc_by_type": { "choice": 13.043478260869565, "noul": -100.0, "score": 30.453846153846154 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 1, "score": 4 }, "score": 0.0 }, "zh": { "accuracy": 0.5, "cc": 47.33675675675678, "cc_by_type": { "choice": -29.729729729729726, "noul": 100.0, "score": 71.74000000000005 }, "low_n": true, "n": 8, "n_by_type": { "choice": 4, "noul": 3, "score": 1 }, "score": 47.33675675675678 } }, "type": { "choice": { "accuracy": 0.336, "cc": 13.974779754707205, "cc_by_type": { "choice": 13.974779754707205 }, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 13.974779754707205 }, "noul": { "accuracy": 0.528, "cc": 5.600000000000005, "cc_by_type": { "noul": 5.600000000000005 }, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 5.600000000000005 }, "score": { "accuracy": 0.32266666666666666, "cc": -8.548239929947377, "cc_by_type": { "score": -8.548239929947377 }, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "usecase": { "advertising": { "accuracy": 0.2916666666666667, "cc": -53.64625846604429, "cc_by_type": { "choice": 11.037891268533771, "noul": 19.999999999999996, "score": -191.97666666666663 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 0.0 }, "code_linting": { "accuracy": 0.25, "cc": -28.865504978177853, "cc_by_type": { "choice": -8.01963993453355, "noul": 19.999999999999996, "score": -98.576875 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 0.0 }, "customer_support": { "accuracy": 0.35, "cc": 2.0286683209263834, "cc_by_type": { "choice": -12.90322580645161, "noul": 25.0, "score": -6.010769230769242 }, "low_n": true, "n": 20, "n_by_type": { "choice": 8, "noul": 8, "score": 4 }, "score": 2.0286683209263834 }, "demand_forecasting": { "accuracy": 0.38461538461538464, "cc": -8.96346355117439, "cc_by_type": { "choice": 3.614457831325295, "noul": -50.0, "score": 19.495151515151534 }, "low_n": true, "n": 26, "n_by_type": { "choice": 11, "noul": 4, "score": 11 }, "score": 0.0 }, "ecommerce_marketplaces": { "accuracy": 0.5, "cc": 24.71274167861124, "cc_by_type": { "choice": 44.099378881987576, "noul": -19.999999999999996, "score": 50.03884615384615 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 5, "score": 6 }, "score": 24.71274167861124 }, "feature_extraction": { "accuracy": 0.27586206896551724, "cc": -17.316809116809114, "cc_by_type": { "choice": 5.9829059829059785, "noul": -19.999999999999996, "score": -37.93333333333333 }, "low_n": true, "n": 29, "n_by_type": { "choice": 15, "noul": 5, "score": 9 }, "score": 0.0 }, "financial_crime": { "accuracy": 0.46153846153846156, "cc": 3.7182956521739166, "cc_by_type": { "choice": 47.82608695652174, "noul": -25.0, "score": -11.671199999999992 }, "low_n": true, "n": 26, "n_by_type": { "choice": 10, "noul": 8, "score": 8 }, "score": 3.7182956521739166 }, "gaming": { "accuracy": 0.27586206896551724, "cc": -26.63343815810153, "cc_by_type": { "choice": 4.950495049504946, "noul": -42.85714285714286, "score": -41.993666666666684 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 7, "score": 11 }, "score": 0.0 }, "insurance_claims": { "accuracy": 0.14285714285714285, "cc": -7.78137566137566, "cc_by_type": { "choice": -26.984126984126984, "noul": -60.0, "score": 63.64000000000001 }, "low_n": true, "n": 14, "n_by_type": { "choice": 8, "noul": 5, "score": 1 }, "score": 0.0 }, "knowledge_graphs": { "accuracy": 0.08695652173913043, "cc": -29.252194167155995, "cc_by_type": { "choice": -28.244274809160306, "noul": -19.999999999999996, "score": -39.512307692307694 }, "low_n": true, "n": 23, "n_by_type": { "choice": 14, "noul": 5, "score": 4 }, "score": 0.0 }, "lead_generation": { "accuracy": 0.36, "cc": 6.393062330623304, "cc_by_type": { "choice": 18.69918699186992, "noul": 0.0, "score": 0.47999999999999154 }, "low_n": true, "n": 25, "n_by_type": { "choice": 8, "noul": 8, "score": 9 }, "score": 6.393062330623304 }, "legal_compliance": { "accuracy": 0.07692307692307693, "cc": -39.32074895531984, "cc_by_type": { "choice": -31.14754098360656, "noul": -100.0, "score": 13.185294117647029 }, "low_n": true, "n": 13, "n_by_type": { "choice": 8, "noul": 1, "score": 4 }, "score": 0.0 }, "llm_guardrails": { "accuracy": 0.4827586206896552, "cc": 11.789371912491177, "cc_by_type": { "choice": 17.431192660550458, "noul": 25.0, "score": -7.0630769230769275 }, "low_n": true, "n": 29, "n_by_type": { "choice": 14, "noul": 8, "score": 7 }, "score": 11.789371912491177 }, "model_routing": { "accuracy": 0.5466666666666666, "cc": 35.63095556818022, "cc_by_type": { "choice": 35.674082015545444, "noul": 63.63636363636365, "score": 7.582421052631572 }, "low_n": false, "n": 150, "n_by_type": { "choice": 74, "noul": 44, "score": 32 }, "score": 35.63095556818022 }, "moderation_trust_safety": { "accuracy": 0.25, "cc": -50.34215538847118, "cc_by_type": { "choice": 36.84210526315789, "noul": -100.0, "score": -87.86857142857143 }, "low_n": true, "n": 16, "n_by_type": { "choice": 8, "noul": 5, "score": 3 }, "score": 0.0 }, "recruiting": { "accuracy": 0.6190476190476191, "cc": 44.91556070953055, "cc_by_type": { "choice": 69.84924623115576, "noul": 66.66666666666667, "score": -1.7692307692307674 }, "low_n": true, "n": 21, "n_by_type": { "choice": 9, "noul": 6, "score": 6 }, "score": 44.91556070953055 }, "risk_assessment": { "accuracy": 0.46153846153846156, "cc": -22.14570230607967, "cc_by_type": { "choice": 43.39622641509434, "noul": -33.333333333333336, "score": -76.50000000000001 }, "low_n": true, "n": 13, "n_by_type": { "choice": 7, "noul": 3, "score": 3 }, "score": 0.0 }, "scientific_discovery": { "accuracy": 0.25, "cc": 0.1523784228166729, "cc_by_type": { "choice": -19.52191235059761, "noul": 14.28571428571428, "score": 5.69333333333335 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 0.1523784228166729 }, "search_retrieval": { "accuracy": 0.36363636363636365, "cc": -15.631133289939976, "cc_by_type": { "choice": 14.08114558472553, "noul": 0.0, "score": -60.974545454545456 }, "low_n": true, "n": 22, "n_by_type": { "choice": 9, "noul": 6, "score": 7 }, "score": 0.0 } } }, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 26.883939686902206, 37.1769715121807 ], "I": [ 3.840627882382685, 11.691929733973469 ], "capability": [ 16.023273303858527, 23.5837139621214 ], "composite": { "A": [ 0.07421330468385162, 1.5542613561325538 ], "B": [ 0.051005668190310965, 1.2102758221999665 ], "C": [ 0.051537017141563626, 1.0793481639809404 ] }, "seed": 18 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": false, "g_med_used": 10.073244581339713, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 32.62324971930184, "capability": 19.21888869011457, "intelligence": 5.814527660927306 }, "per_type": { "choice": { "accuracy": 0.336, "calibration": 14.023911671924271, "cc": 13.974779754707205, "cc_by_type": { "choice": 13.974779754707205 }, "cc_tiered": 14.761056872360381, "cc_tiered_open": 18.56541234955058, "cc_tiered_sealed": 10.956701395170182, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 13.974779754707205 }, "noul": { "accuracy": 0.528, "calibration": 33.12736263736261, "cc": 5.600000000000005, "cc_by_type": { "noul": 5.600000000000005 }, "cc_tiered": 2.6115347276674483, "cc_tiered_open": 1.9710144927536213, "cc_tiered_sealed": 3.2520549625812754, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 5.600000000000005 }, "score": { "accuracy": 0.32266666666666666, "calibration": 50.718474848618655, "cc": -8.548239929947377, "cc_by_type": { "score": -8.548239929947377 }, "cc_tiered": 0.07099138275408867, "cc_tiered_open": 0.14198276550817734, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 30.618919596314058, "capability": 18.126890524291774, "intelligence": 5.634861452269493 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.014666666666666717, "n": 750 }, "noul": { "invalid_rate": 0.029333333333333322, "n": 375 }, "score": { "invalid_rate": 0.01866666666666672, "n": 375 } }, "views": { "Balanced 33:33:33": 0.2059858256814215, "Emphasis on Accuracy 60:20:20": 0.1250037108819809, "Emphasis on Cost 20:20:60": 0.29995214102056916, "Emphasis on Speed 20:60:20": 0.3095514049527884, "Intelligence only": 0.0786327227711625 } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Thiago Silva / @devops_thiago (thi.winnt@gmail.com)", "axes": { "calibration": 48.84618651494331, "cost": 64.96692638610241, "intelligence": 27.338167706853277, "speed": 91.46618934008248 }, "calibration": { "parts": { "choice": { "ece_hard": 0.28518548895899043, "ece_score": 42.96290220820191, "mean_tvd": null, "n": 739, "n_ece_hard": 317, "n_tvd": 0, "score": 42.96290220820191 }, "noul": { "abstention_rate": 0.14010989010989006, "brier": 0.27496240302197805, "ece": 0.25560109890109894, "n": 364, "score": 48.87978021978021 }, "score": { "interval90_coverage": 0.720108695652174, "interval90_width": 0.30865036231884047, "n": 368, "nrps": 0.18476832722826092, "score": 54.69587711684781, "top_ece": 0.3606570652173914 } }, "score": 48.84618651494331 }, "capability": 38.09217711089829, "composite_ci95": { "A": [ 7.891109860006282, 20.334524220090714 ], "B": [ 6.6193128804538945, 17.999509630335773 ], "C": [ 5.479937402782141, 14.121197375063 ] }, "cost": { "basis": "ESTIMATE (*est.), exact base-model market reference; self-hosted open weights with no per-token tariff of their own. The unmodified frozen pricing_v15 resolves the author's own declared base spelling Qwen/Qwen3.5-4B straight through to deepinfra:Qwen/Qwen3.5-4B at USD 0.03 per 1M input and USD 0.15 per 1M output in the dated 25 Sep 2026 snapshot. No new release-lane pricing line.; reference `deepinfra:Qwen/Qwen3.5-4B` frozen at the 25 Sep cut-off, deprecated afterwards (replaced by `Qwen/Qwen3.5-9B`); re-score if it changes (I-3) OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over the item's own candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone. This row's own usage on draw v1.6-regular-20261010-a2: 735,765 input and 0 output tokens over 1,500 decisions. Estimated, not charged.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.014715299999999999 }, "display": "ClassOne Qwen 3.5 4B (Qwen3.5-4B backbone with ClassOne decision heads, schema-constrained single forward pass, self-hosted)", "endpoint_condition": "The author's own README Quickstart install line (pip install -e '.[dev]') at code pin github.com/devops-thiago/class-one@6cca132b and their own serve line (uvicorn classone.server.app:app); their server registers POST /v1/systemone itself, so the request and answer shapes are theirs. CLASSONE_BASE_MODEL was pointed at the LOCAL PINNED SNAPSHOT DIRECTORY, which their own code accepts, so the revision cannot drift to main and classone_heads.pt resolves locally with no request-time hub fetch. Their hard-coded MAX_TOKENS_PER_REQUEST = 8192 refuses longer prompts with HTTP 400; those items are recorded as invalid, the same way for every system. ONE documented mapping of ours and the only one: their server keys score probabilities '1'..'n' while JevBench's canonical keys are '0'..'n-1', so run 58's byte-identical ClassOneReindexAdapter relabels score keys k -> k-1 by calling the unchanged vendored typesafe adapter's own run(), and an item whose key set is not exactly {'1'..'n'} is REFUSED rather than rescored on a guess; choice and noul are untouched. The two already-published ClassOne rows carry exactly this treatment. evaluator-owned Lium RTX 5090 32 GB pod (pod calm-wolf-31, USD 0.72/h; Brasov, Romania, EU by the RIPE whois of its public IP, receipt r90/receipts/POD-IP-WHOIS-R90.txt); pod-level iptables egress cut installed, verified and TIMESTAMPED pod-side (logs/EGRESS-CUT-R90.log) BEFORE the sealed input was uploaded, so the ordering is provable from pod evidence and not only from the session transcript; HF/transformers offline, no credentials in the environment, one decision per request, no batching, no retry. The five rows ran strictly one at a time on the one card, each server started with setsid and stopped by its exact process group with a card-free check in between.", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 36.145599118617284, "I_sealed": 18.530736295089266, "base": 27.338167706853277, "excess": 7.541618242188305, "gap": 17.614862823528018, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 59.400926298363274, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 64.97297297297298, "hard": 41.49103491664046, "judge": 64.40342781806197, "standard": 84.93093344495604 } }, "open|noul": { "cc": 24.24913733609386, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 13.043478260869556, "judge": 10.000000000000009, "standard": 44.44444444444444 } }, "open|score": { "cc": 24.786733721394732, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 39.40090909090911, "hard": 29.234259259259243, "judge": 20.891525423728808, "standard": 14.4274074074074 } }, "sealed|choice": { "cc": 38.6857940545428, "n": { "easy": 72, "hard": 260, "judge": 141, "standard": 127 }, "tiers": { "easy": 45.56262383981645, "hard": 36.68242277673958, "judge": 33.33671553954645, "standard": 47.27773949000691 } }, "sealed|noul": { "cc": 16.457796589375537, "n": { "easy": 39, "hard": 132, "judge": 72, "standard": 57 }, "tiers": { "easy": 38.46153846153846, "hard": -4.545454545454541, "judge": 30.555555555555557, "standard": 26.315789473684205 } }, "sealed|score": { "cc": 0.4486182413494726, "n": { "easy": 33, "hard": 115, "judge": 75, "standard": 77 }, "tiers": { "easy": 28.812056074766346, "hard": -10.255555555555485, "judge": -2.3791150442477926, "standard": 11.916846846846852 } } } }, "jevbench_score": 14.342131800224653, "key": "classone-qwen3.5-4b", "last_measured_on": "2026-10-10", "licence": "apache-2.0", "listing": "ranked", "measured_in": "v1.6.9", "measurement_date_status": "measured", "model_pin": "devops-thiago/classone-qwen3.5-4b@6626fdea7a3e8786bad8d2d5cc8acded1756a17c", "noul_decisive": { "acc_among_decisive": 0.7092651757188498, "decisive": 313, "decisive_rate": 0.8598901098901099, "n": 375, "supported": true, "valid": 364 }, "planned_weight_denominator": 1.0, "rank": 7, "ranked": true, "ranks": { "A": 7, "B": 7, "C": 7, "capability": 11 }, "repo": "https://huggingface.co/devops-thiago/classone-qwen3.5-4b", "scores": { "A": 14.342131800224653, "B": 12.460860262475673, "C": 9.959813750156009 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.23571948995813727, "p50_s_raw": 0.04285974497906864, "p95_s_adjusted": 0.3026812879485078, "p95_s_raw": 0.07634064397425391 }, "status": { "answered_ok": 1471, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns": { "family": { "calibration": { "accuracy": 0.6206896551724138, "cc": 43.73972222222221, "cc_by_type": { "choice": 58.333333333333336, "noul": 33.33333333333333, "score": 39.55249999999998 }, "low_n": true, "n": 29, "n_by_type": { "choice": 16, "noul": 6, "score": 7 }, "score": 43.73972222222221 }, "calibration_gen": { "accuracy": 0.38392857142857145, "cc": -2.796751473164511, "cc_by_type": { "choice": 30.124223602484474, "noul": 7.692307692307687, "score": -46.206785714285694 }, "low_n": false, "n": 112, "n_by_type": { "choice": 56, "noul": 26, "score": 30 }, "score": 0.0 }, "composite_gen": { "accuracy": 0.5510204081632653, "cc": 31.21112378747796, "cc_by_type": { "choice": 48.14814814814815, "noul": 35.71428571428572, "score": 9.770937499999999 }, "low_n": false, "n": 98, "n_by_type": { "choice": 36, "noul": 28, "score": 34 }, "score": 31.21112378747796 }, "fairness": { "accuracy": 0.7777777777777778, "cc": 37.86533333333333, "cc_by_type": { "choice": 100.0, "noul": -66.66666666666667, "score": 80.26266666666666 }, "low_n": true, "n": 27, "n_by_type": { "choice": 12, "noul": 6, "score": 9 }, "score": 37.86533333333333 }, "jaggedness": { "accuracy": 0.42702702702702705, "cc": 3.439985798532349, "cc_by_type": { "choice": 37.47105141269106, "noul": -38.46153846153846, "score": 11.310444444444446 }, "low_n": false, "n": 185, "n_by_type": { "choice": 97, "noul": 39, "score": 49 }, "score": 3.439985798532349 }, "llm_router": { "accuracy": 0.6628787878787878, "cc": 56.02907910025096, "cc_by_type": { "choice": 55.411054342777526, "noul": 53.42465753424656, "score": 59.2515254237288 }, "low_n": false, "n": 264, "n_by_type": { "choice": 131, "noul": 73, "score": 60 }, "score": 56.02907910025096 }, "long_state": { "accuracy": 0.0, "cc": -110.31746031746032, "cc_by_type": { "choice": -30.952380952380945, "noul": -100.0, "score": -200.0 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 11, "score": 7 }, "score": 0.0 }, "ml_usecase": { "accuracy": 0.5161290322580645, "cc": 15.55809364548495, "cc_by_type": { "choice": 26.153846153846153, "noul": 50.0, "score": -29.4795652173913 }, "low_n": false, "n": 31, "n_by_type": { "choice": 15, "noul": 8, "score": 8 }, "score": 15.55809364548495 }, "multilingual": { "accuracy": 0.5015873015873016, "cc": 20.949561234629694, "cc_by_type": { "choice": 36.40841916703987, "noul": 28.735632183908045, "score": -2.2953676470588213 }, "low_n": false, "n": 315, "n_by_type": { "choice": 146, "noul": 87, "score": 82 }, "score": 20.949561234629694 }, "negation_pair_gen": { "accuracy": 0.42857142857142855, "cc": -45.491190476190475, "cc_by_type": { "choice": 28.571428571428573, "noul": 0.0, "score": -165.04500000000002 }, "low_n": false, "n": 56, "n_by_type": { "choice": 28, "noul": 20, "score": 8 }, "score": 0.0 }, "option_order": { "accuracy": 0.6666666666666666, "cc": 57.82463928967813, "cc_by_type": { "choice": 57.82463928967813 }, "low_n": false, "n": 57, "n_by_type": { "choice": 57 }, "score": 57.82463928967813 }, "perturb_pair": { "accuracy": 0.8166666666666667, "cc": 78.77666619437262, "cc_by_type": { "choice": 72.47706422018348, "noul": 85.71428571428572, "score": 78.13864864864864 }, "low_n": false, "n": 60, "n_by_type": { "choice": 34, "noul": 14, "score": 12 }, "score": 78.77666619437262 }, "usecase": { "accuracy": 0.459915611814346, "cc": 13.14458872834252, "cc_by_type": { "choice": 31.778762767392575, "noul": 8.771929824561408, "score": -1.1169264069264218 }, "low_n": false, "n": 237, "n_by_type": { "choice": 111, "noul": 57, "score": 69 }, "score": 13.14458872834252 } }, "lang": { "ar": { "accuracy": 0.6363636363636364, "cc": 13.13722222222224, "cc_by_type": { "choice": 46.666666666666664, "noul": 50.0, "score": -57.254999999999946 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 4, "score": 2 }, "score": 13.13722222222224 }, "cs": { "accuracy": 0.5, "cc": 7.90960451977401, "cc_by_type": { "choice": 49.152542372881356, "noul": -33.333333333333336 }, "low_n": true, "n": 8, "n_by_type": { "choice": 5, "noul": 3 }, "score": 7.90960451977401 }, "da": { "accuracy": 0.5416666666666666, "cc": 36.62549309664694, "cc_by_type": { "choice": 40.8284023668639, "noul": 100.0, "score": -30.951923076923073 }, "low_n": true, "n": 24, "n_by_type": { "choice": 11, "noul": 5, "score": 8 }, "score": 36.62549309664694 }, "de": { "accuracy": 0.5111111111111111, "cc": 22.671439359983808, "cc_by_type": { "choice": 40.25347012673505, "noul": 11.111111111111116, "score": 16.649736842105256 }, "low_n": false, "n": 45, "n_by_type": { "choice": 24, "noul": 9, "score": 12 }, "score": 22.671439359983808 }, "el": { "accuracy": 0.3333333333333333, "cc": -25.492449799196788, "cc_by_type": { "choice": 15.662650602409636, "noul": -19.999999999999996, "score": -72.14 }, "low_n": true, "n": 18, "n_by_type": { "choice": 11, "noul": 5, "score": 2 }, "score": 0.0 }, "en": { "accuracy": 0.5192135835567471, "cc": 21.206639325438474, "cc_by_type": { "choice": 43.690409422321885, "noul": 9.774436090225569, "score": 10.155072463767967 }, "low_n": false, "n": 1119, "n_by_type": { "choice": 577, "noul": 266, "score": 276 }, "score": 21.206639325438474 }, "es": { "accuracy": 0.7142857142857143, "cc": 55.06090909090909, "cc_by_type": { "choice": 47.63636363636363, "noul": 100.0, "score": 17.546363636363637 }, "low_n": true, "n": 28, "n_by_type": { "choice": 10, "noul": 11, "score": 7 }, "score": 55.06090909090909 }, "fi": { "accuracy": 0.4444444444444444, "cc": 24.4131455399061, "cc_by_type": { "choice": 15.49295774647887, "noul": 33.33333333333333 }, "low_n": true, "n": 9, "n_by_type": { "choice": 6, "noul": 3 }, "score": 24.4131455399061 }, "fr": { "accuracy": 0.6, "cc": 26.589757539122846, "cc_by_type": { "choice": 74.61212976022568, "noul": 0.0, "score": 5.15714285714286 }, "low_n": true, "n": 15, "n_by_type": { "choice": 5, "noul": 4, "score": 6 }, "score": 26.589757539122846 }, "hi": { "accuracy": 0.4, "cc": 9.455742753623186, "cc_by_type": { "choice": 13.043478260869566, "noul": 19.999999999999996, "score": -4.676250000000004 }, "low_n": false, "n": 30, "n_by_type": { "choice": 11, "noul": 10, "score": 9 }, "score": 9.455742753623186 }, "id": { "accuracy": 0.16666666666666666, "cc": -14.516129032258062, "cc_by_type": { "choice": -29.032258064516125, "noul": 0.0 }, "low_n": true, "n": 6, "n_by_type": { "choice": 4, "noul": 2 }, "score": 0.0 }, "it": { "accuracy": 0.7647058823529411, "cc": 59.36320992007523, "cc_by_type": { "choice": 74.61212976022568, "noul": 50.0, "score": 53.4775 }, "low_n": true, "n": 17, "n_by_type": { "choice": 5, "noul": 4, "score": 8 }, "score": 59.36320992007523 }, "ja": { "accuracy": 0.3125, "cc": 1.1244367015098795, "cc_by_type": { "choice": 2.4390243902439046, "noul": -14.28571428571429, "score": 15.220000000000022 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 1.1244367015098795 }, "ko": { "accuracy": 0.7272727272727273, "cc": 77.836, "cc_by_type": { "choice": 100.0, "noul": 100.0, "score": 33.507999999999996 }, "low_n": true, "n": 11, "n_by_type": { "choice": 4, "noul": 2, "score": 5 }, "score": 77.836 }, "mixed": { "accuracy": 1.0, "cc": 98.28333333333332, "cc_by_type": { "choice": 100.0, "noul": 100.0, "score": 94.84999999999997 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 4, "score": 1 }, "score": 98.28333333333332 }, "nl": { "accuracy": 0.3076923076923077, "cc": -7.494183844011142, "cc_by_type": { "choice": -0.2785515320334245, "noul": 0.0, "score": -22.204 }, "low_n": true, "n": 13, "n_by_type": { "choice": 5, "noul": 6, "score": 2 }, "score": 0.0 }, "no": { "accuracy": 0.5714285714285714, "cc": 33.87978142076503, "cc_by_type": { "choice": 34.42622950819673, "noul": 33.33333333333333 }, "low_n": true, "n": 7, "n_by_type": { "choice": 4, "noul": 3 }, "score": 33.87978142076503 }, "pl": { "accuracy": 0.575, "cc": 30.731736262935428, "cc_by_type": { "choice": 68.51770878880629, "noul": 60.00000000000001, "score": -36.32250000000001 }, "low_n": false, "n": 40, "n_by_type": { "choice": 16, "noul": 10, "score": 14 }, "score": 30.731736262935428 }, "pt": { "accuracy": 0.7222222222222222, "cc": 60.06590480178671, "cc_by_type": { "choice": 81.90954773869345, "noul": 66.66666666666667, "score": 31.62150000000001 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 6, "score": 5 }, "score": 60.06590480178671 }, "sv": { "accuracy": 0.5, "cc": 0.943703703703702, "cc_by_type": { "choice": 11.111111111111109, "noul": 50.0, "score": -58.28 }, "low_n": true, "n": 8, "n_by_type": { "choice": 3, "noul": 4, "score": 1 }, "score": 0.943703703703702 }, "tr": { "accuracy": 0.45454545454545453, "cc": 3.5850292397660724, "cc_by_type": { "choice": 47.368421052631575, "noul": 33.33333333333333, "score": -69.94666666666669 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 3, "score": 3 }, "score": 3.5850292397660724 }, "uk": { "accuracy": 0.09090909090909091, "cc": -59.95774804905239, "cc_by_type": { "choice": -30.43478260869565, "noul": -100.0, "score": -49.43846153846154 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 1, "score": 4 }, "score": 0.0 }, "zh": { "accuracy": 1.0, "cc": 69.91333333333333, "cc_by_type": { "choice": 100.00000000000001, "noul": 100.0, "score": 9.739999999999993 }, "low_n": true, "n": 8, "n_by_type": { "choice": 4, "noul": 3, "score": 1 }, "score": 69.91333333333333 } }, "type": { "choice": { "accuracy": 0.5573333333333333, "cc": 42.6498531698048, "cc_by_type": { "choice": 42.6498531698048 }, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 42.6498531698048 }, "noul": { "accuracy": 0.592, "cc": 18.399999999999995, "cc_by_type": { "noul": 18.399999999999995 }, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 18.399999999999995 }, "score": { "accuracy": 0.384, "cc": 6.636094570928175, "cc_by_type": { "score": 6.636094570928175 }, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 6.636094570928175 } }, "usecase": { "advertising": { "accuracy": 0.4166666666666667, "cc": -33.318480688266526, "cc_by_type": { "choice": 11.037891268533771, "noul": 19.999999999999996, "score": -130.99333333333334 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 0.0 }, "code_linting": { "accuracy": 0.2916666666666667, "cc": 0.8938591107473973, "cc_by_type": { "choice": 1.8003273322422266, "noul": 60.00000000000001, "score": -59.11875000000004 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 0.8938591107473973 }, "customer_support": { "accuracy": 0.5, "cc": 25.121935483870967, "cc_by_type": { "choice": 3.225806451612906, "noul": 25.0, "score": 47.13999999999999 }, "low_n": true, "n": 20, "n_by_type": { "choice": 8, "noul": 8, "score": 4 }, "score": 25.121935483870967 }, "demand_forecasting": { "accuracy": 0.34615384615384615, "cc": 2.214011196300356, "cc_by_type": { "choice": 3.614457831325295, "noul": 0.0, "score": 3.0275757575757734 }, "low_n": true, "n": 26, "n_by_type": { "choice": 11, "noul": 4, "score": 11 }, "score": 2.214011196300356 }, "ecommerce_marketplaces": { "accuracy": 0.4444444444444444, "cc": 10.85620321707278, "cc_by_type": { "choice": 44.099378881987576, "noul": -19.999999999999996, "score": 8.469230769230762 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 5, "score": 6 }, "score": 10.85620321707278 }, "feature_extraction": { "accuracy": 0.27586206896551724, "cc": -14.564156954156951, "cc_by_type": { "choice": 23.076923076923077, "noul": -60.0, "score": -6.769393939393931 }, "low_n": true, "n": 29, "n_by_type": { "choice": 15, "noul": 5, "score": 9 }, "score": 0.0 }, "financial_crime": { "accuracy": 0.38461538461538464, "cc": 6.2751362318840584, "cc_by_type": { "choice": 34.78260869565218, "noul": 0.0, "score": -15.957200000000004 }, "low_n": true, "n": 26, "n_by_type": { "choice": 10, "noul": 8, "score": 8 }, "score": 6.2751362318840584 }, "gaming": { "accuracy": 0.4482758620689655, "cc": 11.478893603646071, "cc_by_type": { "choice": 28.712871287128706, "noul": 42.85714285714286, "score": -37.133333333333354 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 7, "score": 11 }, "score": 11.478893603646071 }, "insurance_claims": { "accuracy": 0.2857142857142857, "cc": -53.55068783068785, "cc_by_type": { "choice": 36.507936507936506, "noul": -100.0, "score": -97.16000000000005 }, "low_n": true, "n": 14, "n_by_type": { "choice": 8, "noul": 5, "score": 1 }, "score": 0.0 }, "knowledge_graphs": { "accuracy": 0.43478260869565216, "cc": 7.427903699354076, "cc_by_type": { "choice": 26.717557251908392, "noul": 19.999999999999996, "score": -24.43384615384616 }, "low_n": true, "n": 23, "n_by_type": { "choice": 14, "noul": 5, "score": 4 }, "score": 7.427903699354076 }, "lead_generation": { "accuracy": 0.52, "cc": 20.725170731707326, "cc_by_type": { "choice": 51.21951219512196, "noul": 25.0, "score": -14.043999999999969 }, "low_n": true, "n": 25, "n_by_type": { "choice": 8, "noul": 8, "score": 9 }, "score": 20.725170731707326 }, "legal_compliance": { "accuracy": 0.15384615384615385, "cc": -29.474503375120538, "cc_by_type": { "choice": -14.754098360655735, "noul": -100.0, "score": 26.330588235294115 }, "low_n": true, "n": 13, "n_by_type": { "choice": 8, "noul": 1, "score": 4 }, "score": 0.0 }, "llm_guardrails": { "accuracy": 0.5172413793103449, "cc": 22.654965890378737, "cc_by_type": { "choice": 44.95412844036697, "noul": 25.0, "score": -1.9892307692307654 }, "low_n": true, "n": 29, "n_by_type": { "choice": 14, "noul": 8, "score": 7 }, "score": 22.654965890378737 }, "model_routing": { "accuracy": 0.7333333333333333, "cc": 62.21502323863556, "cc_by_type": { "choice": 67.83704100777273, "noul": 68.18181818181819, "score": 50.626210526315795 }, "low_n": false, "n": 150, "n_by_type": { "choice": 74, "noul": 44, "score": 32 }, "score": 62.21502323863556 }, "moderation_trust_safety": { "accuracy": 0.6875, "cc": 29.053859649122796, "cc_by_type": { "choice": 52.63157894736842, "noul": 100.0, "score": -65.47000000000003 }, "low_n": true, "n": 16, "n_by_type": { "choice": 8, "noul": 5, "score": 3 }, "score": 29.053859649122796 }, "recruiting": { "accuracy": 0.7142857142857143, "cc": 37.555988059957905, "cc_by_type": { "choice": 69.84924623115576, "noul": 33.33333333333333, "score": 9.485384615384618 }, "low_n": true, "n": 21, "n_by_type": { "choice": 9, "noul": 6, "score": 6 }, "score": 37.555988059957905 }, "risk_assessment": { "accuracy": 0.46153846153846156, "cc": -1.4943134171907815, "cc_by_type": { "choice": 43.39622641509434, "noul": 33.33333333333333, "score": -81.21250000000002 }, "low_n": true, "n": 13, "n_by_type": { "choice": 7, "noul": 3, "score": 3 }, "score": 0.0 }, "scientific_discovery": { "accuracy": 0.40625, "cc": 17.67869126667931, "cc_by_type": { "choice": 12.350597609561751, "noul": 42.85714285714286, "score": -2.171666666666683 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 17.67869126667931 }, "search_retrieval": { "accuracy": 0.5909090909090909, "cc": 36.62560810973699, "cc_by_type": { "choice": 42.72076372315036, "noul": 33.33333333333333, "score": 33.82272727272727 }, "low_n": true, "n": 22, "n_by_type": { "choice": 9, "noul": 6, "score": 7 }, "score": 36.62560810973699 } } }, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 43.368446520267796, 53.193644571845724 ], "I": [ 21.59802304927416, 31.44548663789592 ], "capability": [ 33.4245046846555, 41.60857704511476 ], "composite": { "A": [ 7.891109860006282, 20.334524220090714 ], "B": [ 6.6193128804538945, 17.999509630335773 ], "C": [ 5.479937402782141, 14.121197375063 ] }, "seed": 19 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": false, "g_med_used": 10.073244581339713, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 48.84618651494331, "capability": 38.09217711089829, "intelligence": 27.338167706853277 }, "per_type": { "choice": { "accuracy": 0.5573333333333333, "calibration": 42.96290220820191, "cc": 42.6498531698048, "cc_by_type": { "choice": 42.6498531698048 }, "cc_tiered": 49.043360176453035, "cc_tiered_open": 59.400926298363274, "cc_tiered_sealed": 38.6857940545428, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 42.6498531698048 }, "noul": { "accuracy": 0.592, "calibration": 48.87978021978021, "cc": 18.399999999999995, "cc_by_type": { "noul": 18.399999999999995 }, "cc_tiered": 20.3534669627347, "cc_tiered_open": 24.24913733609386, "cc_tiered_sealed": 16.457796589375537, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 18.399999999999995 }, "score": { "accuracy": 0.384, "calibration": 54.69587711684781, "cc": 6.636094570928175, "cc_by_type": { "score": 6.636094570928175 }, "cc_tiered": 12.617675981372102, "cc_tiered_open": 24.786733721394732, "cc_tiered_sealed": 0.4486182413494726, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 6.636094570928175 } }, "raw_on_A": { "calibration": 44.37411603270306, "capability": 35.42499096258019, "intelligence": 26.47586589245732 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.014666666666666717, "n": 750 }, "noul": { "invalid_rate": 0.029333333333333322, "n": 375 }, "score": { "invalid_rate": 0.01866666666666672, "n": 375 } }, "views": { "Balanced 33:33:33": 14.257369523966407, "Emphasis on Accuracy 60:20:20": 10.98579554132928, "Emphasis on Cost 20:20:60": 15.954348902584416, "Emphasis on Speed 20:60:20": 17.632944190866688, "Intelligence only": 8.172749758440709 } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Cloudflare", "axes": { "calibration": 82.15469329918429, "cost": 26.792900075221908, "intelligence": 34.47716016566848, "speed": 86.7880661703893 }, "calibration": { "parts": { "choice": { "ece_hard": 0.09821189024390234, "ece_score": 80.35762195121954, "mean_tvd": null, "n": 750, "n_ece_hard": 328, "n_tvd": 0, "score": 80.35762195121954 }, "noul": { "abstention_rate": 0.3493333333333334, "brier": 0.1572640221333334, "ece": 0.06652586666666661, "n": 375, "score": 86.69482666666669 }, "score": { "interval90_coverage": 0.8506666666666667, "interval90_width": 0.36177777777777775, "n": 375, "nrps": 0.11951350773999989, "score": 79.41163127966668, "top_ece": 0.14612693333333326 } }, "score": 82.15469329918429 }, "capability": 58.315926732426384, "composite_ci95": { "A": [ 3.8823605492743414, 8.586118919617187 ], "B": [ 3.562206076177268, 8.318090708030555 ], "C": [ 2.696083714773848, 5.9625825830674914 ] }, "cost": { "basis": "ESTIMATE (*est.), base-class reference. Self-hosted open weights; the declared base Qwen/Qwen3-Omni-30B-A3B-Instruct is not a key of any frozen pricing table. Method as the v1.6.5 Liquid rows: the nearest same-family entry of the frozen 25 Sep 2026 snapshot, deepinfra:Qwen/Qwen3-VL-30B-A3B-Instruct (the multimodal Qwen3 30B-A3B), at USD 0.15 per 1M input and USD 0.60 per 1M output. The authors' systemone() has no generation path, so output_tokens is a literal 0. This row's own usage on draw v1.6-regular-20261010-a2: 2,755,730 input and 0 output tokens over 1,500 decisions.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.275573 }, "display": "Clef-omni", "endpoint_condition": "evaluator-owned Lium RTX PRO 6000 Blackwell 96 GB pod (Beltsville US, 29 min, USD 1.19/h); the authors' own load_release_model() + systemone() from the pinned repo's joint_schema_model.py behind the run-39 loopback shim (127.0.0.1, single-threaded, no answer arithmetic), pointed at the LOCAL PINNED SNAPSHOT DIRECTORY, defaults unchanged (max_length 64,000). Stack as the model card tested: torch 2.11.0+cu130, transformers 5.10.2, plus torchvision 0.26.0+cu130, which the Omni video processor requires to load. HF/transformers offline, no credentials, pod-level iptables egress cut installed and verified before the sealed input was uploaded. All 17 LFS files verified against the Hugging Face tree digests. GPU memory 62,729 MiB after load and at most 74,735 MiB during the run (per-minute log), of 97,887 MiB.", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 37.25444425506104, "I_sealed": 31.699876076275928, "base": 34.47716016566848, "excess": -4.518676402554602, "gap": 5.554568178785111, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 65.02240890008018, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 84.43243243243245, "hard": 67.9144385026738, "judge": 56.49307844429796, "standard": 62.32733361239012 } }, "open|noul": { "cc": 10.814354727398204, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": -13.043478260869568, "judge": 0.0, "standard": 44.44444444444444 } }, "open|score": { "cc": 35.92656913770473, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 51.82848484848485, "hard": 42.1251851851852, "judge": 49.07338983050846, "standard": -4.141851851851852 } }, "sealed|choice": { "cc": 53.547209374932514, "n": { "easy": 72, "hard": 260, "judge": 141, "standard": 127 }, "tiers": { "easy": 62.45698195849411, "hard": 49.543805650214345, "judge": 52.513824767896104, "standard": 58.64920744314267 } }, "sealed|noul": { "cc": 9.67795362532205, "n": { "easy": 39, "hard": 132, "judge": 72, "standard": 57 }, "tiers": { "easy": 43.58974358974359, "hard": -10.606060606060607, "judge": 16.666666666666675, "standard": 22.807017543859654 } }, "sealed|score": { "cc": 31.87446522857323, "n": { "easy": 33, "hard": 115, "judge": 75, "standard": 77 }, "tiers": { "easy": 45.32728971962617, "hard": 22.845027777777847, "judge": 36.40668141592918, "standard": 36.408603603603595 } } } }, "jevbench_score": 6.06640325953113, "key": "clef-omni", "last_measured_on": "2026-10-10", "licence": "Apache-2.0", "listing": "ranked", "measured_in": "v1.6.6", "measurement_date_status": "measured", "model_pin": "Cloudflare/clef-omni@0db1cd2607d76a7bdb2a382f659e7b313079f84b", "noul_decisive": { "acc_among_decisive": 0.8565573770491803, "decisive": 244, "decisive_rate": 0.6506666666666666, "n": 375, "supported": true, "valid": 375 }, "planned_weight_denominator": 1.0, "rank": 9, "ranked": true, "ranks": { "A": 9, "B": 9, "C": 9, "capability": 6 }, "repo": "https://huggingface.co/Cloudflare/clef-omni", "scores": { "A": 6.06640325953113, "B": 5.735167687307061, "C": 4.2127800413410625 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.43618169901892545, "p50_s_raw": 0.14309084950946271, "p95_s_adjusted": 0.4803147715982049, "p95_s_raw": 0.16515738579910247 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns": { "family": { "calibration": { "accuracy": 0.896551724137931, "cc": 86.57097222222222, "cc_by_type": { "choice": 91.66666666666667, "noul": 100.0, "score": 68.04625 }, "low_n": true, "n": 29, "n_by_type": { "choice": 16, "noul": 6, "score": 7 }, "score": 86.57097222222222 }, "calibration_gen": { "accuracy": 0.5982142857142857, "cc": 32.91760471412644, "cc_by_type": { "choice": 55.745341614906835, "noul": 15.384615384615374, "score": 27.622857142857114 }, "low_n": false, "n": 112, "n_by_type": { "choice": 56, "noul": 26, "score": 30 }, "score": 32.91760471412644 }, "composite_gen": { "accuracy": 0.6632653061224489, "cc": 50.659661596119925, "cc_by_type": { "choice": 74.07407407407408, "noul": 35.71428571428572, "score": 42.19062499999998 }, "low_n": false, "n": 98, "n_by_type": { "choice": 36, "noul": 28, "score": 34 }, "score": 50.659661596119925 }, "fairness": { "accuracy": 0.7407407407407407, "cc": 51.41265359477125, "cc_by_type": { "choice": 88.23529411764707, "noul": 0.0, "score": 66.00266666666667 }, "low_n": true, "n": 27, "n_by_type": { "choice": 12, "noul": 6, "score": 9 }, "score": 51.41265359477125 }, "jaggedness": { "accuracy": 0.42162162162162165, "cc": -0.8300322403138493, "cc_by_type": { "choice": 30.523390458545624, "noul": -53.84615384615385, "score": 20.832666666666675 }, "low_n": false, "n": 185, "n_by_type": { "choice": 97, "noul": 39, "score": 49 }, "score": 0.0 }, "llm_router": { "accuracy": 0.6818181818181818, "cc": 57.872019428772546, "cc_by_type": { "choice": 50.76637250348352, "noul": 53.42465753424656, "score": 69.42502824858757 }, "low_n": false, "n": 264, "n_by_type": { "choice": 131, "noul": 73, "score": 60 }, "score": 57.872019428772546 }, "long_state": { "accuracy": 0.3103448275862069, "cc": -1.3167821067821077, "cc_by_type": { "choice": 16.66666666666667, "noul": -27.27272727272727, "score": 6.655714285714276 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 11, "score": 7 }, "score": 0.0 }, "ml_usecase": { "accuracy": 0.5483870967741935, "cc": 5.659007803790407, "cc_by_type": { "choice": 53.84615384615384, "noul": -25.0, "score": -11.869130434782615 }, "low_n": false, "n": 31, "n_by_type": { "choice": 15, "noul": 8, "score": 8 }, "score": 5.659007803790407 }, "multilingual": { "accuracy": 0.5714285714285714, "cc": 29.241308106053285, "cc_by_type": { "choice": 54.32154052843709, "noul": 5.747126436781613, "score": 27.655257352941153 }, "low_n": false, "n": 315, "n_by_type": { "choice": 146, "noul": 87, "score": 82 }, "score": 29.241308106053285 }, "negation_pair_gen": { "accuracy": 0.6607142857142857, "cc": 18.345555555555563, "cc_by_type": { "choice": 66.66666666666667, "noul": 19.999999999999996, "score": -31.62999999999998 }, "low_n": false, "n": 56, "n_by_type": { "choice": 28, "noul": 20, "score": 8 }, "score": 18.345555555555563 }, "option_order": { "accuracy": 0.8947368421052632, "cc": 86.68146503884572, "cc_by_type": { "choice": 86.68146503884572 }, "low_n": false, "n": 57, "n_by_type": { "choice": 57 }, "score": 86.68146503884572 }, "perturb_pair": { "accuracy": 0.9, "cc": 81.10219498659866, "cc_by_type": { "choice": 84.27260812581913, "noul": 71.42857142857142, "score": 87.6054054054054 }, "low_n": false, "n": 60, "n_by_type": { "choice": 34, "noul": 14, "score": 12 }, "score": 81.10219498659866 }, "usecase": { "accuracy": 0.5232067510548524, "cc": 17.11602027853846, "cc_by_type": { "choice": 53.74831374060514, "noul": -15.789473684210531, "score": 13.389220779220778 }, "low_n": false, "n": 237, "n_by_type": { "choice": 111, "noul": 57, "score": 69 }, "score": 17.11602027853846 } }, "lang": { "ar": { "accuracy": 0.45454545454545453, "cc": -23.164444444444445, "cc_by_type": { "choice": 46.666666666666664, "noul": -50.0, "score": -66.16 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 4, "score": 2 }, "score": 0.0 }, "cs": { "accuracy": 0.5, "cc": 7.90960451977401, "cc_by_type": { "choice": 49.152542372881356, "noul": -33.333333333333336 }, "low_n": true, "n": 8, "n_by_type": { "choice": 5, "noul": 3 }, "score": 7.90960451977401 }, "da": { "accuracy": 0.5416666666666666, "cc": 23.599881656804737, "cc_by_type": { "choice": 52.662721893491124, "noul": -19.999999999999996, "score": 38.13692307692308 }, "low_n": true, "n": 24, "n_by_type": { "choice": 11, "noul": 5, "score": 8 }, "score": 23.599881656804737 }, "de": { "accuracy": 0.7111111111111111, "cc": 51.80671574665221, "cc_by_type": { "choice": 72.84248642124321, "noul": 55.55555555555556, "score": 27.02210526315789 }, "low_n": false, "n": 45, "n_by_type": { "choice": 24, "noul": 9, "score": 12 }, "score": 51.80671574665221 }, "el": { "accuracy": 0.3333333333333333, "cc": -55.7336546184739, "cc_by_type": { "choice": 39.7590361445783, "noul": -100.0, "score": -106.96 }, "low_n": true, "n": 18, "n_by_type": { "choice": 11, "noul": 5, "score": 2 }, "score": 0.0 }, "en": { "accuracy": 0.6094727435210009, "cc": 34.60248155699925, "cc_by_type": { "choice": 56.70218732473357, "noul": 9.774436090225569, "score": 37.33082125603862 }, "low_n": false, "n": 1119, "n_by_type": { "choice": 577, "noul": 266, "score": 276 }, "score": 34.60248155699925 }, "es": { "accuracy": 0.7142857142857143, "cc": 54.17545454545455, "cc_by_type": { "choice": 73.81818181818181, "noul": 63.63636363636365, "score": 25.071818181818184 }, "low_n": true, "n": 28, "n_by_type": { "choice": 10, "noul": 11, "score": 7 }, "score": 54.17545454545455 }, "fi": { "accuracy": 0.1111111111111111, "cc": -52.816901408450704, "cc_by_type": { "choice": -5.63380281690141, "noul": -100.0 }, "low_n": true, "n": 9, "n_by_type": { "choice": 6, "noul": 3 }, "score": 0.0 }, "fr": { "accuracy": 0.6, "cc": 17.07420198356729, "cc_by_type": { "choice": 74.61212976022568, "noul": -50.0, "score": 26.610476190476174 }, "low_n": true, "n": 15, "n_by_type": { "choice": 5, "noul": 4, "score": 6 }, "score": 17.07420198356729 }, "hi": { "accuracy": 0.6333333333333333, "cc": 44.937223084886135, "cc_by_type": { "choice": 62.73291925465839, "noul": 60.00000000000001, "score": 12.078749999999994 }, "low_n": false, "n": 30, "n_by_type": { "choice": 11, "noul": 10, "score": 9 }, "score": 44.937223084886135 }, "id": { "accuracy": 0.16666666666666666, "cc": -48.387096774193544, "cc_by_type": { "choice": 3.225806451612906, "noul": -100.0 }, "low_n": true, "n": 6, "n_by_type": { "choice": 4, "noul": 2 }, "score": 0.0 }, "it": { "accuracy": 0.7058823529411765, "cc": 50.644668253408554, "cc_by_type": { "choice": 74.61212976022568, "noul": 50.0, "score": 27.321874999999995 }, "low_n": true, "n": 17, "n_by_type": { "choice": 5, "noul": 4, "score": 8 }, "score": 50.644668253408554 }, "ja": { "accuracy": 0.46875, "cc": 18.159518002322883, "cc_by_type": { "choice": 26.829268292682926, "noul": -14.28571428571429, "score": 41.93500000000001 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 18.159518002322883 }, "ko": { "accuracy": 0.6363636363636364, "cc": 66.1930382513661, "cc_by_type": { "choice": 67.21311475409836, "noul": 100.0, "score": 31.365999999999982 }, "low_n": true, "n": 11, "n_by_type": { "choice": 4, "noul": 2, "score": 5 }, "score": 66.1930382513661 }, "mixed": { "accuracy": 1.0, "cc": 85.82333333333332, "cc_by_type": { "choice": 100.0, "noul": 100.0, "score": 57.469999999999956 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 4, "score": 1 }, "score": 85.82333333333332 }, "nl": { "accuracy": 0.5384615384615384, "cc": 15.964908077994421, "cc_by_type": { "choice": 49.860724233983284, "noul": 0.0, "score": -1.9660000000000233 }, "low_n": true, "n": 13, "n_by_type": { "choice": 5, "noul": 6, "score": 2 }, "score": 15.964908077994421 }, "no": { "accuracy": 0.2857142857142857, "cc": -15.846994535519126, "cc_by_type": { "choice": 1.6393442622950836, "noul": -33.333333333333336 }, "low_n": true, "n": 7, "n_by_type": { "choice": 4, "noul": 3 }, "score": 0.0 }, "pl": { "accuracy": 0.575, "cc": 36.12173351608311, "cc_by_type": { "choice": 84.25885439440316, "noul": 19.999999999999996, "score": 4.106346153846174 }, "low_n": false, "n": 40, "n_by_type": { "choice": 16, "noul": 10, "score": 14 }, "score": 36.12173351608311 }, "pt": { "accuracy": 0.8333333333333334, "cc": 74.19423813512005, "cc_by_type": { "choice": 81.90954773869345, "noul": 66.66666666666667, "score": 74.0065 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 6, "score": 5 }, "score": 74.19423813512005 }, "sv": { "accuracy": 0.375, "cc": 4.158888888888879, "cc_by_type": { "choice": -33.333333333333336, "noul": 50.0, "score": -4.190000000000027 }, "low_n": true, "n": 8, "n_by_type": { "choice": 3, "noul": 4, "score": 1 }, "score": 4.158888888888879 }, "tr": { "accuracy": 0.7272727272727273, "cc": 22.35695906432748, "cc_by_type": { "choice": 73.6842105263158, "noul": 33.33333333333333, "score": -39.94666666666669 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 3, "score": 3 }, "score": 22.35695906432748 }, "uk": { "accuracy": 0.7272727272727273, "cc": 12.17157190635451, "cc_by_type": { "choice": 78.26086956521739, "noul": -100.0, "score": 58.25384615384615 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 1, "score": 4 }, "score": 12.17157190635451 }, "zh": { "accuracy": 0.625, "cc": 50.52282282282281, "cc_by_type": { "choice": 35.13513513513514, "noul": 33.33333333333333, "score": 83.09999999999995 }, "low_n": true, "n": 8, "n_by_type": { "choice": 4, "noul": 3, "score": 1 }, "score": 50.52282282282281 } }, "type": { "choice": { "accuracy": 0.6653333333333333, "cc": 56.64190706512352, "cc_by_type": { "choice": 56.64190706512352 }, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 56.64190706512352 }, "noul": { "accuracy": 0.5573333333333333, "cc": 11.466666666666669, "cc_by_type": { "noul": 11.466666666666669 }, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 11.466666666666669 }, "score": { "accuracy": 0.5333333333333333, "cc": 33.64992119089322, "cc_by_type": { "score": 33.64992119089322 }, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 33.64992119089322 } }, "usecase": { "advertising": { "accuracy": 0.5416666666666666, "cc": 8.511753615229727, "cc_by_type": { "choice": 40.691927512355846, "noul": -60.0, "score": 44.843333333333334 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 8.511753615229727 }, "code_linting": { "accuracy": 0.5833333333333334, "cc": 9.050032733224224, "cc_by_type": { "choice": 70.54009819967267, "noul": -19.999999999999996, "score": -23.39 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 9.050032733224224 }, "customer_support": { "accuracy": 0.3, "cc": -13.059735318444998, "cc_by_type": { "choice": 35.483870967741936, "noul": -75.0, "score": 0.336923076923068 }, "low_n": true, "n": 20, "n_by_type": { "choice": 8, "noul": 8, "score": 4 }, "score": 0.0 }, "demand_forecasting": { "accuracy": 0.38461538461538464, "cc": -8.969614214433495, "cc_by_type": { "choice": 39.7590361445783, "noul": -50.0, "score": -16.667878787878788 }, "low_n": true, "n": 26, "n_by_type": { "choice": 11, "noul": 4, "score": 11 }, "score": 0.0 }, "ecommerce_marketplaces": { "accuracy": 0.5555555555555556, "cc": 47.39004937091894, "cc_by_type": { "choice": 44.099378881987576, "noul": 60.00000000000001, "score": 38.07076923076923 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 5, "score": 6 }, "score": 47.39004937091894 }, "feature_extraction": { "accuracy": 0.4827586206896552, "cc": 12.539619269619266, "cc_by_type": { "choice": 48.71794871794871, "noul": -60.0, "score": 48.90090909090908 }, "low_n": true, "n": 29, "n_by_type": { "choice": 15, "noul": 5, "score": 9 }, "score": 12.539619269619266 }, "financial_crime": { "accuracy": 0.5384615384615384, "cc": 14.042347826086965, "cc_by_type": { "choice": 73.91304347826087, "noul": 0.0, "score": -31.78599999999998 }, "low_n": true, "n": 26, "n_by_type": { "choice": 10, "noul": 8, "score": 8 }, "score": 14.042347826086965 }, "gaming": { "accuracy": 0.41379310344827586, "cc": 5.618274556027025, "cc_by_type": { "choice": 28.712871287128706, "noul": -14.28571428571429, "score": 2.4276666666666613 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 7, "score": 11 }, "score": 5.618274556027025 }, "insurance_claims": { "accuracy": 0.2857142857142857, "cc": -11.208359788359795, "cc_by_type": { "choice": 20.634920634920636, "noul": -100.0, "score": 45.73999999999998 }, "low_n": true, "n": 14, "n_by_type": { "choice": 8, "noul": 5, "score": 1 }, "score": 0.0 }, "knowledge_graphs": { "accuracy": 0.5217391304347826, "cc": 9.780414954002737, "cc_by_type": { "choice": 45.03816793893129, "noul": -60.0, "score": 44.30307692307692 }, "low_n": true, "n": 23, "n_by_type": { "choice": 14, "noul": 5, "score": 4 }, "score": 9.780414954002737 }, "lead_generation": { "accuracy": 0.4, "cc": 5.368834688346889, "cc_by_type": { "choice": 83.73983739837398, "noul": -75.0, "score": 7.3666666666666885 }, "low_n": true, "n": 25, "n_by_type": { "choice": 8, "noul": 8, "score": 9 }, "score": 5.368834688346889 }, "legal_compliance": { "accuracy": 0.3076923076923077, "cc": -20.515570556091294, "cc_by_type": { "choice": 34.42622950819673, "noul": -100.0, "score": 4.027058823529394 }, "low_n": true, "n": 13, "n_by_type": { "choice": 8, "noul": 1, "score": 4 }, "score": 0.0 }, "llm_guardrails": { "accuracy": 0.6206896551724138, "cc": 32.95168666196189, "cc_by_type": { "choice": 63.30275229357799, "noul": 0.0, "score": 35.55230769230767 }, "low_n": true, "n": 29, "n_by_type": { "choice": 14, "noul": 8, "score": 7 }, "score": 32.95168666196189 }, "model_routing": { "accuracy": 0.74, "cc": 63.43091944512483, "cc_by_type": { "choice": 69.44518895738408, "noul": 77.27272727272727, "score": 43.57484210526314 }, "low_n": false, "n": 150, "n_by_type": { "choice": 74, "noul": 44, "score": 32 }, "score": 63.43091944512483 }, "moderation_trust_safety": { "accuracy": 0.5, "cc": 24.84468671679197, "cc_by_type": { "choice": 21.052631578947366, "noul": -19.999999999999996, "score": 73.48142857142854 }, "low_n": true, "n": 16, "n_by_type": { "choice": 8, "noul": 5, "score": 3 }, "score": 24.84468671679197 }, "recruiting": { "accuracy": 0.6190476190476191, "cc": 25.69128720525705, "cc_by_type": { "choice": 69.84924623115576, "noul": 0.0, "score": 7.2246153846153955 }, "low_n": true, "n": 21, "n_by_type": { "choice": 9, "noul": 6, "score": 6 }, "score": 25.69128720525705 }, "risk_assessment": { "accuracy": 0.5384615384615384, "cc": 14.524019916142551, "cc_by_type": { "choice": 43.39622641509434, "noul": 33.33333333333333, "score": -33.15750000000002 }, "low_n": true, "n": 13, "n_by_type": { "choice": 7, "noul": 3, "score": 3 }, "score": 14.524019916142551 }, "scientific_discovery": { "accuracy": 0.3125, "cc": -9.035004163241226, "cc_by_type": { "choice": 28.286852589641434, "noul": -42.85714285714286, "score": -12.53472222222225 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 0.0 }, "search_retrieval": { "accuracy": 0.8181818181818182, "cc": 63.02598587305031, "cc_by_type": { "choice": 71.36038186157518, "noul": 66.66666666666667, "score": 51.05090909090908 }, "low_n": true, "n": 22, "n_by_type": { "choice": 9, "noul": 6, "score": 7 }, "score": 63.02598587305031 } } }, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 75.99498334146031, 83.8879680508817 ], "I": [ 28.570607529410697, 40.0809890185093 ], "capability": [ 53.56177237020907, 61.07325750582405 ], "composite": { "A": [ 3.8823605492743414, 8.586118919617187 ], "B": [ 3.562206076177268, 8.318090708030555 ], "C": [ 2.696083714773848, 5.9625825830674914 ] }, "seed": 20 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": false, "g_med_used": 10.073244581339713, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 82.15469329918429, "capability": 58.315926732426384, "intelligence": 34.47716016566848 }, "per_type": { "choice": { "accuracy": 0.6653333333333333, "calibration": 80.35762195121954, "cc": 56.64190706512352, "cc_by_type": { "choice": 56.64190706512352 }, "cc_tiered": 59.28480913750634, "cc_tiered_open": 65.02240890008018, "cc_tiered_sealed": 53.547209374932514, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 56.64190706512352 }, "noul": { "accuracy": 0.5573333333333333, "calibration": 86.69482666666669, "cc": 11.466666666666669, "cc_by_type": { "noul": 11.466666666666669 }, "cc_tiered": 10.246154176360127, "cc_tiered_open": 10.814354727398204, "cc_tiered_sealed": 9.67795362532205, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 11.466666666666669 }, "score": { "accuracy": 0.5333333333333333, "calibration": 79.41163127966668, "cc": 33.64992119089322, "cc_by_type": { "score": 33.64992119089322 }, "cc_tiered": 33.90051718313898, "cc_tiered_open": 35.92656913770473, "cc_tiered_sealed": 31.87446522857323, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 33.64992119089322 } }, "raw_on_A": { "calibration": 80.93316891446167, "capability": 59.00548965209646, "intelligence": 37.07781038973123 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 5.261179060414329, "Emphasis on Accuracy 60:20:20": 5.02460048275502, "Emphasis on Cost 20:20:60": 4.476423118785528, "Emphasis on Speed 20:60:20": 6.765856602997514, "Intelligence only": 4.70710504931535 } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Omar / @kouhxp (hi@kouh.me)", "axes": { "calibration": 77.50040036924202, "cost": 78.3542548810428, "intelligence": 8.600978834309654, "speed": 72.8877498600171 }, "calibration": { "parts": { "choice": { "ece_hard": 0.08639168454258676, "ece_score": 82.72166309148264, "mean_tvd": null, "n": 739, "n_ece_hard": 317, "n_tvd": 0, "score": 82.72166309148264 }, "noul": { "abstention_rate": 0.706043956043956, "brier": 0.24853405620583507, "ece": 0.12309816483516484, "n": 364, "score": 75.38036703296703 }, "score": { "interval90_coverage": 0.8695652173913043, "interval90_width": 0.6397192028985506, "n": 368, "nrps": 0.16768646620403746, "score": 74.39917098327638, "top_ece": 0.17216505706521745 } }, "score": 77.50040036924202 }, "capability": 43.05068960177584, "composite_ci95": { "A": [ 0.35718114749129054, 1.8750127120910802 ], "B": [ 0.2489508080898962, 1.3935931889582263 ], "C": [ 0.24804246353561837, 1.3020921611743612 ] }, "cost": { "basis": "ESTIMATE (*est.), input tokens only. The author's gutsy-inference 0.3.0 server emits no output-token field in any spelling; under the chief-of-staff decision of 9 Oct 2026 a missing output_tokens field on a native option-probability model that generates no text is treated as 0 output tokens, exactly as a server reporting a literal 0 is treated. Zero output is a verified property of this backend, not an assumption: it has no generation path at all (no generate/create_completion/n_predict/max_tokens/sampler); it prefills via llm.eval(tokens) and reads llama_get_logits_ith(ctx,-1) (r52 PREREG-AMENDMENT-4). Rate: the declared base Qwen/Qwen3.5-0.8B resolves in the frozen 25 Sep 2026 snapshot to deepinfra:Qwen/Qwen3.5-0.8B at USD 0.01 per 1M input and USD 0.05 per 1M output. This row's own measured usage on draw v1.6-regular-20261010-a2: 789,980 input tokens and 0 output over 1,500 decisions => USD 0.0052665333333333335 per 1,000 decisions.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0052665333333333335 }, "display": "Gutsy 0.8B v0.3", "endpoint_condition": "evaluator-owned Lium pod (2xRTX3090 node, Germany), CPU-only by the author's own config (n_gpu_layers 0), 32 vCPU, egress cut with iptables and proven before the sealed input was uploaded; the author's own gutsy-inference 0.3.0 with their models.example.json settings unchanged (n_ctx 8192, reject_slot true, n_threads null)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 9.070301583187167, "I_sealed": 8.131656085432141, "base": 8.600978834309654, "excess": -9.134599083584687, "gap": 0.9386454977550258, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 25.412389147780978, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 37.72972972972974, "hard": 20.729789241899976, "judge": 12.986156888595916, "standard": 47.25826705734617 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 14.28571428571428, "hard": -47.82608695652174, "judge": -60.0, "standard": -33.333333333333336 } }, "open|score": { "cc": 1.7985156017805268, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 48.240515151515154, "hard": -7.74469814814811, "judge": 1.9498728813559296, "standard": -2.5630925925926196 } }, "sealed|choice": { "cc": 24.394968256296423, "n": { "easy": 72, "hard": 260, "judge": 141, "standard": 127 }, "tiers": { "easy": 23.036813014912926, "hard": 20.853028470924478, "judge": 26.031150119222776, "standard": 29.70365265334253 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 39, "hard": 132, "judge": 72, "standard": 57 }, "tiers": { "easy": -43.58974358974359, "hard": -78.78787878787878, "judge": -47.22222222222222, "standard": -61.40350877192983 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 33, "hard": 115, "judge": 75, "standard": 77 }, "tiers": { "easy": 12.014685981308393, "hard": -2.484919999999935, "judge": -12.305056637168143, "standard": 12.255661711711696 } } } }, "jevbench_score": 0.7604367004496922, "key": "gutsy-0.8b-v0.3", "last_measured_on": "2026-10-10", "listing": "ranked", "measured_in": "v1.6.5", "measurement_date_status": "measured", "model_pin": "kouhxp/gutsy@d325eb1742cc1c200e04b24417f192a0dbf74f91", "noul_decisive": { "acc_among_decisive": 0.7383177570093458, "decisive": 107, "decisive_rate": 0.29395604395604397, "n": 375, "supported": true, "valid": 364 }, "planned_weight_denominator": 1.0, "rank": 12, "ranked": true, "ranks": { "A": 12, "B": 12, "C": 12, "capability": 9 }, "repo": "https://huggingface.co/kouhxp/gutsy", "scores": { "A": 0.7604367004496922, "B": 0.5441131491342173, "C": 0.5280810419789529 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 1.6613200119999418, "p50_s_raw": 0.755660005999971, "p95_s_adjusted": 3.0957915995999423, "p95_s_raw": 1.4728957997999712 }, "status": { "answered_ok": 1471, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns": { "family": { "calibration": { "accuracy": 0.5862068965517241, "cc": 30.238494444444445, "cc_by_type": { "choice": 58.333333333333336, "noul": 0.0, "score": 32.38215000000001 }, "low_n": true, "n": 29, "n_by_type": { "choice": 16, "noul": 6, "score": 7 }, "score": 30.238494444444445 }, "calibration_gen": { "accuracy": 0.38392857142857145, "cc": -19.57648651457238, "cc_by_type": { "choice": 41.770186335403736, "noul": -92.3076923076923, "score": -8.191953571428567 }, "low_n": false, "n": 112, "n_by_type": { "choice": 56, "noul": 26, "score": 30 }, "score": 0.0 }, "composite_gen": { "accuracy": 0.2653061224489796, "cc": -24.789154706790118, "cc_by_type": { "choice": 29.62962962962963, "noul": -100.0, "score": -3.9970937499999915 }, "low_n": false, "n": 98, "n_by_type": { "choice": 36, "noul": 28, "score": 34 }, "score": 0.0 }, "fairness": { "accuracy": 0.5555555555555556, "cc": 21.924596601307183, "cc_by_type": { "choice": 52.94117647058823, "noul": 0.0, "score": 12.83261333333332 }, "low_n": true, "n": 27, "n_by_type": { "choice": 12, "noul": 6, "score": 9 }, "score": 21.924596601307183 }, "jaggedness": { "accuracy": 0.2756756756756757, "cc": -23.766898037764125, "cc_by_type": { "choice": 9.680407596109312, "noul": -79.48717948717949, "score": -1.4939222222221993 }, "low_n": false, "n": 185, "n_by_type": { "choice": 97, "noul": 39, "score": 49 }, "score": 0.0 }, "llm_router": { "accuracy": 0.4583333333333333, "cc": 21.153469908954868, "cc_by_type": { "choice": 26.61402693915468, "noul": 1.3698630136986356, "score": 35.47651977401129 }, "low_n": false, "n": 264, "n_by_type": { "choice": 131, "noul": 73, "score": 60 }, "score": 21.153469908954868 }, "long_state": { "accuracy": 0.0, "cc": -110.31746031746032, "cc_by_type": { "choice": -30.952380952380945, "noul": -100.0, "score": -200.0 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 11, "score": 7 }, "score": 0.0 }, "ml_usecase": { "accuracy": 0.25806451612903225, "cc": -38.646075919732446, "cc_by_type": { "choice": 26.153846153846153, "noul": -100.0, "score": -42.09207391304348 }, "low_n": false, "n": 31, "n_by_type": { "choice": 15, "noul": 8, "score": 8 }, "score": 0.0 }, "multilingual": { "accuracy": 0.346031746031746, "cc": -10.63069831663855, "cc_by_type": { "choice": 30.138826690550832, "noul": -58.62068965517242, "score": -3.4102319852940655 }, "low_n": false, "n": 315, "n_by_type": { "choice": 146, "noul": 87, "score": 82 }, "score": 0.0 }, "negation_pair_gen": { "accuracy": 0.26785714285714285, "cc": -26.649703571428574, "cc_by_type": { "choice": 14.285714285714286, "noul": -90.0, "score": -4.234825000000009 }, "low_n": false, "n": 56, "n_by_type": { "choice": 28, "noul": 20, "score": 8 }, "score": 0.0 }, "option_order": { "accuracy": 0.17543859649122806, "cc": -4.328523862375131, "cc_by_type": { "choice": -4.328523862375131 }, "low_n": false, "n": 57, "n_by_type": { "choice": 57 }, "score": 0.0 }, "perturb_pair": { "accuracy": 0.5166666666666667, "cc": 19.74922901184277, "cc_by_type": { "choice": 41.02228047182176, "noul": -28.57142857142857, "score": 46.79683513513512 }, "low_n": false, "n": 60, "n_by_type": { "choice": 34, "noul": 14, "score": 12 }, "score": 19.74922901184277 }, "usecase": { "accuracy": 0.3206751054852321, "cc": -16.990773799131347, "cc_by_type": { "choice": 27.153594141453095, "noul": -75.43859649122807, "score": -2.6873190476190745 }, "low_n": false, "n": 237, "n_by_type": { "choice": 111, "noul": 57, "score": 69 }, "score": 0.0 } }, "lang": { "ar": { "accuracy": 0.36363636363636365, "cc": -27.022149999999996, "cc_by_type": { "choice": 20.000000000000004, "noul": -50.0, "score": -51.06644999999999 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 4, "score": 2 }, "score": 0.0 }, "cs": { "accuracy": 0.5, "cc": -12.711864406779661, "cc_by_type": { "choice": 74.57627118644068, "noul": -100.0 }, "low_n": true, "n": 8, "n_by_type": { "choice": 5, "noul": 3 }, "score": 0.0 }, "da": { "accuracy": 0.20833333333333334, "cc": -32.610591814595644, "cc_by_type": { "choice": 5.325443786982243, "noul": -100.0, "score": -3.1572192307691926 }, "low_n": true, "n": 24, "n_by_type": { "choice": 11, "noul": 5, "score": 8 }, "score": 0.0 }, "de": { "accuracy": 0.4666666666666667, "cc": 2.2274148960461893, "cc_by_type": { "choice": 40.25347012673505, "noul": -33.333333333333336, "score": -0.23789210526314886 }, "low_n": false, "n": 45, "n_by_type": { "choice": 24, "noul": 9, "score": 12 }, "score": 2.2274148960461893 }, "el": { "accuracy": 0.2222222222222222, "cc": -51.29191646586346, "cc_by_type": { "choice": 15.662650602409636, "noul": -100.0, "score": -69.53840000000002 }, "low_n": true, "n": 18, "n_by_type": { "choice": 11, "noul": 5, "score": 2 }, "score": 0.0 }, "en": { "accuracy": 0.34405719392314565, "cc": -11.632396312202536, "cc_by_type": { "choice": 22.602355580482268, "noul": -61.654135338345874, "score": 4.154590821255999 }, "low_n": false, "n": 1119, "n_by_type": { "choice": 577, "noul": 266, "score": 276 }, "score": 0.0 }, "es": { "accuracy": 0.5, "cc": 13.68897878787879, "cc_by_type": { "choice": 60.72727272727273, "noul": -27.27272727272727, "score": 7.612390909090916 }, "low_n": true, "n": 28, "n_by_type": { "choice": 10, "noul": 11, "score": 7 }, "score": 13.68897878787879 }, "fi": { "accuracy": 0.2222222222222222, "cc": -42.25352112676057, "cc_by_type": { "choice": 15.49295774647887, "noul": -100.0 }, "low_n": true, "n": 9, "n_by_type": { "choice": 6, "noul": 3 }, "score": 0.0 }, "fr": { "accuracy": 0.3333333333333333, "cc": -15.758361508496193, "cc_by_type": { "choice": 74.61212976022568, "noul": -100.0, "score": -21.887214285714272 }, "low_n": true, "n": 15, "n_by_type": { "choice": 5, "noul": 4, "score": 6 }, "score": 0.0 }, "hi": { "accuracy": 0.3, "cc": -16.665771829710142, "cc_by_type": { "choice": 13.043478260869566, "noul": -60.0, "score": -3.0407937499999926 }, "low_n": false, "n": 30, "n_by_type": { "choice": 11, "noul": 10, "score": 9 }, "score": 0.0 }, "id": { "accuracy": 0.0, "cc": -64.51612903225806, "cc_by_type": { "choice": -29.032258064516125, "noul": -100.0 }, "low_n": true, "n": 6, "n_by_type": { "choice": 4, "noul": 2 }, "score": 0.0 }, "it": { "accuracy": 0.6470588235294118, "cc": 27.99738442348378, "cc_by_type": { "choice": 49.22425952045134, "noul": 0.0, "score": 34.76789375 }, "low_n": true, "n": 17, "n_by_type": { "choice": 5, "noul": 4, "score": 8 }, "score": 27.99738442348378 }, "ja": { "accuracy": 0.3125, "cc": -23.373968770567163, "cc_by_type": { "choice": 18.69918699186992, "noul": -71.42857142857143, "score": -17.39252187499998 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 0.0 }, "ko": { "accuracy": 0.2727272727272727, "cc": -31.130543497267748, "cc_by_type": { "choice": 34.42622950819673, "noul": -100.0, "score": -27.81785999999997 }, "low_n": true, "n": 11, "n_by_type": { "choice": 4, "noul": 2, "score": 5 }, "score": 0.0 }, "mixed": { "accuracy": 0.36363636363636365, "cc": -3.6486553113553115, "cc_by_type": { "choice": 34.06593406593406, "noul": -50.0, "score": 4.988100000000006 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 4, "score": 1 }, "score": 0.0 }, "nl": { "accuracy": 0.46153846153846156, "cc": -0.1424452553389107, "cc_by_type": { "choice": 49.860724233983284, "noul": 0.0, "score": -50.288060000000016 }, "low_n": true, "n": 13, "n_by_type": { "choice": 5, "noul": 6, "score": 2 }, "score": 0.0 }, "no": { "accuracy": 0.42857142857142855, "cc": 0.5464480874316955, "cc_by_type": { "choice": 34.42622950819673, "noul": -33.333333333333336 }, "low_n": true, "n": 7, "n_by_type": { "choice": 4, "noul": 3 }, "score": 0.5464480874316955 }, "pl": { "accuracy": 0.4, "cc": 12.299376097900057, "cc_by_type": { "choice": 60.64713598600787, "noul": -19.999999999999996, "score": -3.7490076923077043 }, "low_n": false, "n": 40, "n_by_type": { "choice": 16, "noul": 10, "score": 14 }, "score": 12.299376097900057 }, "pt": { "accuracy": 0.5, "cc": 26.514546072026793, "cc_by_type": { "choice": 45.7286432160804, "noul": 0.0, "score": 33.81499499999999 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 6, "score": 5 }, "score": 26.514546072026793 }, "sv": { "accuracy": 0.0, "cc": -56.24097777777778, "cc_by_type": { "choice": -33.333333333333336, "noul": -100.0, "score": -35.3896 }, "low_n": true, "n": 8, "n_by_type": { "choice": 3, "noul": 4, "score": 1 }, "score": 0.0 }, "tr": { "accuracy": 0.45454545454545453, "cc": 0.7387181286549614, "cc_by_type": { "choice": 47.368421052631575, "noul": 33.33333333333333, "score": -78.48560000000002 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 3, "score": 3 }, "score": 0.7387181286549614 }, "uk": { "accuracy": 0.09090909090909091, "cc": -46.02425072463768, "cc_by_type": { "choice": -8.695652173913041, "noul": -100.0, "score": -29.3771 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 1, "score": 4 }, "score": 0.0 }, "zh": { "accuracy": 0.125, "cc": -50.39295435435435, "cc_by_type": { "choice": -29.729729729729726, "noul": -33.333333333333336, "score": -88.11580000000001 }, "low_n": true, "n": 8, "n_by_type": { "choice": 4, "noul": 3, "score": 1 }, "score": 0.0 } }, "type": { "choice": { "accuracy": 0.41733333333333333, "cc": 24.512005527724998, "cc_by_type": { "choice": 24.512005527724998 }, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 24.512005527724998 }, "noul": { "accuracy": 0.21066666666666667, "cc": -57.86666666666667, "cc_by_type": { "noul": -57.86666666666667 }, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.3466666666666667, "cc": 0.9913860770578653, "cc_by_type": { "score": 0.9913860770578653 }, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.9913860770578653 } }, "usecase": { "advertising": { "accuracy": 0.3333333333333333, "cc": -20.792305967417175, "cc_by_type": { "choice": 30.807248764415156, "noul": -60.0, "score": -33.184166666666684 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 0.0 }, "code_linting": { "accuracy": 0.16666666666666666, "cc": -51.04649872817785, "cc_by_type": { "choice": -8.01963993453355, "noul": -60.0, "score": -85.11985625 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 0.0 }, "customer_support": { "accuracy": 0.4, "cc": 7.8360928866832085, "cc_by_type": { "choice": 35.483870967741936, "noul": -50.0, "score": 38.02440769230768 }, "low_n": true, "n": 20, "n_by_type": { "choice": 8, "noul": 8, "score": 4 }, "score": 7.8360928866832085 }, "demand_forecasting": { "accuracy": 0.23076923076923078, "cc": -39.86741871729342, "cc_by_type": { "choice": -8.43373493975904, "noul": -100.0, "score": -11.168521212121219 }, "low_n": true, "n": 26, "n_by_type": { "choice": 11, "noul": 4, "score": 11 }, "score": 0.0 }, "ecommerce_marketplaces": { "accuracy": 0.2777777777777778, "cc": -17.362241654722098, "cc_by_type": { "choice": 44.099378881987576, "noul": -100.0, "score": 3.8138961538461302 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 5, "score": 6 }, "score": 0.0 }, "feature_extraction": { "accuracy": 0.1724137931034483, "cc": -29.604258430458433, "cc_by_type": { "choice": -2.564102564102566, "noul": -60.0, "score": -26.248672727272737 }, "low_n": true, "n": 29, "n_by_type": { "choice": 15, "noul": 5, "score": 9 }, "score": 0.0 }, "financial_crime": { "accuracy": 0.38461538461538464, "cc": -8.077263014492752, "cc_by_type": { "choice": 47.82608695652174, "noul": -100.0, "score": 27.942124 }, "low_n": true, "n": 26, "n_by_type": { "choice": 10, "noul": 8, "score": 8 }, "score": 0.0 }, "gaming": { "accuracy": 0.27586206896551724, "cc": -33.71579672167217, "cc_by_type": { "choice": 16.831683168316832, "noul": -100.0, "score": -17.97907333333333 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 7, "score": 11 }, "score": 0.0 }, "insurance_claims": { "accuracy": 0.14285714285714285, "cc": -70.40589841269842, "cc_by_type": { "choice": 4.761904761904763, "noul": -100.0, "score": -115.9796 }, "low_n": true, "n": 14, "n_by_type": { "choice": 8, "noul": 5, "score": 1 }, "score": 0.0 }, "knowledge_graphs": { "accuracy": 0.21739130434782608, "cc": -16.350229066353496, "cc_by_type": { "choice": -9.923664122137408, "noul": -60.0, "score": 20.872976923076926 }, "low_n": true, "n": 23, "n_by_type": { "choice": 14, "noul": 5, "score": 4 }, "score": 0.0 }, "lead_generation": { "accuracy": 0.36, "cc": -2.498773089430904, "cc_by_type": { "choice": 83.73983739837398, "noul": -75.0, "score": -16.23615666666669 }, "low_n": true, "n": 25, "n_by_type": { "choice": 8, "noul": 8, "score": 9 }, "score": 0.0 }, "legal_compliance": { "accuracy": 0.38461538461538464, "cc": -26.543207810993252, "cc_by_type": { "choice": 34.42622950819673, "noul": -100.0, "score": -14.055852941176482 }, "low_n": true, "n": 13, "n_by_type": { "choice": 8, "noul": 1, "score": 4 }, "score": 0.0 }, "llm_guardrails": { "accuracy": 0.3793103448275862, "cc": -5.703711032698185, "cc_by_type": { "choice": 44.95412844036697, "noul": -50.0, "score": -12.065261538461524 }, "low_n": true, "n": 29, "n_by_type": { "choice": 14, "noul": 8, "score": 7 }, "score": 0.0 }, "model_routing": { "accuracy": 0.5, "cc": 20.346316406896474, "cc_by_type": { "choice": 42.106673813990895, "noul": 9.090909090909083, "score": 9.841366315789447 }, "low_n": false, "n": 150, "n_by_type": { "choice": 74, "noul": 44, "score": 32 }, "score": 20.346316406896474 }, "moderation_trust_safety": { "accuracy": 0.3125, "cc": -21.78233634085213, "cc_by_type": { "choice": 36.84210526315789, "noul": -100.0, "score": -2.1891142857142887 }, "low_n": true, "n": 16, "n_by_type": { "choice": 8, "noul": 5, "score": 3 }, "score": 0.0 }, "recruiting": { "accuracy": 0.38095238095238093, "cc": -7.3411204741657015, "cc_by_type": { "choice": 54.77386934673366, "noul": -100.0, "score": 23.20276923076924 }, "low_n": true, "n": 21, "n_by_type": { "choice": 9, "noul": 6, "score": 6 }, "score": 0.0 }, "risk_assessment": { "accuracy": 0.5384615384615384, "cc": 28.47671559224318, "cc_by_type": { "choice": 62.264150943396224, "noul": 33.33333333333333, "score": -10.167337500000006 }, "low_n": true, "n": 13, "n_by_type": { "choice": 7, "noul": 3, "score": 3 }, "score": 28.47671559224318 }, "scientific_discovery": { "accuracy": 0.21875, "cc": -30.3334952412572, "cc_by_type": { "choice": 12.350597609561751, "noul": -100.0, "score": -3.35108333333336 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 0.0 }, "search_retrieval": { "accuracy": 0.18181818181818182, "cc": -40.61265147175815, "cc_by_type": { "choice": 14.08114558472553, "noul": -100.0, "score": -35.91909999999998 }, "low_n": true, "n": 22, "n_by_type": { "choice": 9, "noul": 6, "score": 7 }, "score": 0.0 } } }, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 71.1160109805681, 79.80133322822977 ], "I": [ 6.552688842251461, 11.980624454535823 ], "capability": [ 39.46993021223222, 45.1260255721779 ], "composite": { "A": [ 0.35718114749129054, 1.8750127120910802 ], "B": [ 0.2489508080898962, 1.3935931889582263 ], "C": [ 0.24804246353561837, 1.3020921611743612 ] }, "seed": 21 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": false, "g_med_used": 10.073244581339713, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 77.50040036924202, "capability": 43.05068960177584, "intelligence": 8.600978834309654 }, "per_type": { "choice": { "accuracy": 0.41733333333333333, "calibration": 82.72166309148264, "cc": 24.512005527724998, "cc_by_type": { "choice": 24.512005527724998 }, "cc_tiered": 24.9036787020387, "cc_tiered_open": 25.412389147780978, "cc_tiered_sealed": 24.394968256296423, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 24.512005527724998 }, "noul": { "accuracy": 0.21066666666666667, "calibration": 75.38036703296703, "cc": -57.86666666666667, "cc_by_type": { "noul": -57.86666666666667 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.3466666666666667, "calibration": 74.39917098327638, "cc": 0.9913860770578653, "cc_by_type": { "score": 0.9913860770578653 }, "cc_tiered": 0.8992578008902634, "cc_tiered_open": 1.7985156017805268, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.9913860770578653 } }, "raw_on_A": { "calibration": 78.45530493357067, "capability": 43.54816426637622, "intelligence": 8.641023599181757 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.014666666666666717, "n": 750 }, "noul": { "invalid_rate": 0.029333333333333322, "n": 375 }, "score": { "invalid_rate": 0.01866666666666672, "n": 375 } }, "views": { "Balanced 33:33:33": 0.6218800588368827, "Emphasis on Accuracy 60:20:20": 0.39424898336691744, "Emphasis on Cost 20:20:60": 0.8792467571253542, "Emphasis on Speed 20:60:20": 0.8693565458528719, "Intelligence only": 0.25450928339078893 } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "eljiwo / jiwidi (fhjaime96@gmail.com)", "axes": { "calibration": 78.54210067220151, "cost": 78.40361168685378, "intelligence": 15.872104931583502, "speed": 94.33479301471377 }, "calibration": { "parts": { "choice": { "ece_hard": 0.14318770225758568, "ece_score": 71.36245954848286, "mean_tvd": null, "n": 739, "n_ece_hard": 317, "n_tvd": 0, "score": 71.36245954848286 }, "noul": { "abstention_rate": 0.6428571428571428, "brier": 0.22635226125046812, "ece": 0.11375593371020894, "n": 364, "score": 77.24881325795822 }, "score": { "interval90_coverage": 0.9646739130434783, "interval90_width": 0.731884057971014, "n": 368, "nrps": 0.13949242305496334, "score": 87.01502921016349, "top_ece": 0.060103496370883344 } }, "score": 78.54210067220151 }, "capability": 47.20710280189251, "composite_ci95": { "A": [ 2.071813842220017, 6.329337632277144 ], "B": [ 1.533726731769256, 4.973696434425022 ], "C": [ 1.4387596126527897, 4.395373355748017 ] }, "cost": { "basis": "ESTIMATE (*est.), exact base-model market reference; self-hosted open weights with no per-token tariff of their own. The unmodified frozen pricing_v15 resolves the author's own declared base spelling Qwen/Qwen3.5-0.8B straight through to deepinfra:Qwen/Qwen3.5-0.8B at USD 0.01 per 1M input and USD 0.05 per 1M output in the dated 25 Sep 2026 snapshot. No new release-lane pricing line. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over the item's own candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone. This row's own usage on draw v1.6-regular-20261010-a2: 786,993 input and 0 output tokens over 1,500 decisions. Estimated, not charged.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0052466200000000004 }, "display": "jiwo 0.8b (Qwen3.5-0.8B backbone with a trained decision head, one forward pass, native probabilities, self-hosted)", "endpoint_condition": "The author's own install line from their model card (pip install 'jiwo[cuda] @ git+https://github.com/jiwidi/jiwo@8716abab') and their own `jiwo serve`, nothing patched. The server reads JIWO_CHECKPOINT and calls from_pretrained with NO revision and there is no JIWO_REVISION setting, so it was pointed at the LOCAL PINNED SNAPSHOT DIRECTORY and the served revision cannot have drifted to main. JIWO_MAX_LENGTH=65536, the author's own documented value for their published runs: the 8,192 default refuses longer prompts with HTTP 400 and never truncates. The trained context is 8,192 tokens, so the longest items of this draw are answered outside the range the model saw in training. CUDA graphs are OFF (JIWO_CUDA_GRAPHS unset): the author's own card says graphs agree with eager on only 99.7 % of fields, so eager is the exact path and the author's 14.5 ms headline uses a mode this measurement does not use. Requests are serialised by the author's own asyncio.Lock. evaluator-owned Lium RTX 5090 32 GB pod (pod calm-wolf-31, USD 0.72/h; Brasov, Romania, EU by the RIPE whois of its public IP, receipt r90/receipts/POD-IP-WHOIS-R90.txt); pod-level iptables egress cut installed, verified and TIMESTAMPED pod-side (logs/EGRESS-CUT-R90.log) BEFORE the sealed input was uploaded, so the ordering is provable from pod evidence and not only from the session transcript; HF/transformers offline, no credentials in the environment, one decision per request, no batching, no retry. The five rows ran strictly one at a time on the one card, each server started with setsid and stopped by its exact process group with a card-free check in between.", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 20.53318982032003, "I_sealed": 11.211020042846975, "base": 15.872104931583502, "excess": -0.7510748038666595, "gap": 9.322169777473054, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 35.509112308155075, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 41.62162162162163, "hard": 30.16671909405474, "judge": 32.76203032300594, "standard": 47.25826705734617 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -14.28571428571429, "hard": -39.13043478260869, "judge": -30.000000000000004, "standard": -33.333333333333336 } }, "open|score": { "cc": 26.09045715280502, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 50.4799625425419, "hard": 18.20775252838498, "judge": 31.25102514541178, "standard": 21.920261717866516 } }, "sealed|choice": { "cc": 24.483755785527055, "n": { "easy": 72, "hard": 260, "judge": 141, "standard": 127 }, "tiers": { "easy": 32.4225675252894, "hard": 18.37968561064086, "judge": 25.117954441682315, "standard": 31.771192281185392 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 39, "hard": 132, "judge": 72, "standard": 57 }, "tiers": { "easy": -17.948717948717952, "hard": -69.6969696969697, "judge": -33.333333333333336, "standard": -54.385964912280706 } }, "sealed|score": { "cc": 9.149304343013874, "n": { "easy": 33, "hard": 115, "judge": 75, "standard": 77 }, "tiers": { "easy": 26.922451761233834, "hard": 2.576868911219099, "judge": 8.09382448748016, "standard": 14.99082128079402 } } } }, "jevbench_score": 4.067767731552672, "key": "jiwo-0.8b", "last_measured_on": "2026-10-10", "licence": "Apache-2.0 (weights, model card); MIT (code, github.com/jiwidi/jiwo)", "listing": "ranked", "measured_in": "v1.6.9", "measurement_date_status": "measured", "model_pin": "eljiwo/jiwo-0.8b@52d217f8d68114e853b31ca364f8cb7da1b3c64d", "noul_decisive": { "acc_among_decisive": 0.7615384615384615, "decisive": 130, "decisive_rate": 0.35714285714285715, "n": 375, "supported": true, "valid": 364 }, "planned_weight_denominator": 1.0, "rank": 10, "ranked": true, "ranks": { "A": 10, "B": 10, "C": 10, "capability": 8 }, "repo": "https://huggingface.co/eljiwo/jiwo-0.8b", "scores": { "A": 4.067767731552672, "B": 3.1083596840941525, "C": 2.8248387024671335 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.18979715087916701, "p50_s_raw": 0.01989857543958351, "p95_s_adjusted": 0.19419185474980621, "p95_s_raw": 0.022095927374903113 }, "status": { "answered_ok": 1471, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns": { "family": { "calibration": { "accuracy": 0.5172413793103449, "cc": 24.45957255405951, "cc_by_type": { "choice": 66.66666666666667, "noul": 0.0, "score": 6.712050995511854 }, "low_n": true, "n": 29, "n_by_type": { "choice": 16, "noul": 6, "score": 7 }, "score": 24.45957255405951 }, "calibration_gen": { "accuracy": 0.35714285714285715, "cc": -9.724218679588965, "cc_by_type": { "choice": 25.46583850931677, "noul": -61.53846153846154, "score": 6.89996699037787 }, "low_n": false, "n": 112, "n_by_type": { "choice": 56, "noul": 26, "score": 30 }, "score": 0.0 }, "composite_gen": { "accuracy": 0.37755102040816324, "cc": -5.9291709848934415, "cc_by_type": { "choice": 40.74074074074074, "noul": -78.57142857142857, "score": 20.04317487600751 }, "low_n": false, "n": 98, "n_by_type": { "choice": 36, "noul": 28, "score": 34 }, "score": 0.0 }, "fairness": { "accuracy": 0.7777777777777778, "cc": 71.6287638691846, "cc_by_type": { "choice": 64.70588235294117, "noul": 100.0, "score": 50.1804092546126 }, "low_n": true, "n": 27, "n_by_type": { "choice": 12, "noul": 6, "score": 9 }, "score": 71.6287638691846 }, "jaggedness": { "accuracy": 0.24324324324324326, "cc": -31.490478394830276, "cc_by_type": { "choice": -0.04631773969430275, "noul": -94.87179487179486, "score": 0.446677426998332 }, "low_n": false, "n": 185, "n_by_type": { "choice": 97, "noul": 39, "score": 49 }, "score": 0.0 }, "llm_router": { "accuracy": 0.48863636363636365, "cc": 31.193268142476445, "cc_by_type": { "choice": 28.471899674872283, "noul": 12.328767123287676, "score": 52.77913762926938 }, "low_n": false, "n": 264, "n_by_type": { "choice": 131, "noul": 73, "score": 60 }, "score": 31.193268142476445 }, "long_state": { "accuracy": 0.0, "cc": -110.31746031746032, "cc_by_type": { "choice": -30.952380952380945, "noul": -100.0, "score": -200.0 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 11, "score": 7 }, "score": 0.0 }, "ml_usecase": { "accuracy": 0.16129032258064516, "cc": -35.935636537648094, "cc_by_type": { "choice": -1.5384615384615408, "noul": -100.0, "score": -6.268448074482746 }, "low_n": false, "n": 31, "n_by_type": { "choice": 15, "noul": 8, "score": 8 }, "score": 0.0 }, "multilingual": { "accuracy": 0.37142857142857144, "cc": -0.3712463104306612, "cc_by_type": { "choice": 29.243170622480974, "noul": -47.12643678160919, "score": 16.769527227836235 }, "low_n": false, "n": 315, "n_by_type": { "choice": 146, "noul": 87, "score": 82 }, "score": 0.0 }, "negation_pair_gen": { "accuracy": 0.35714285714285715, "cc": -7.457195719833385, "cc_by_type": { "choice": 14.285714285714286, "noul": -40.0, "score": 3.3426985547855614 }, "low_n": false, "n": 56, "n_by_type": { "choice": 28, "noul": 20, "score": 8 }, "score": 0.0 }, "option_order": { "accuracy": 0.49122807017543857, "cc": 35.62708102108768, "cc_by_type": { "choice": 35.62708102108768 }, "low_n": false, "n": 57, "n_by_type": { "choice": 57 }, "score": 35.62708102108768 }, "perturb_pair": { "accuracy": 0.6, "cc": 31.852412487051, "cc_by_type": { "choice": 52.81782437745741, "noul": -14.28571428571429, "score": 57.02512736940989 }, "low_n": false, "n": 60, "n_by_type": { "choice": 34, "noul": 14, "score": 12 }, "score": 31.852412487051 }, "usecase": { "accuracy": 0.29957805907172996, "cc": -18.97090524037614, "cc_by_type": { "choice": 29.46617845442283, "noul": -82.45614035087719, "score": -3.9227538246740634 }, "low_n": false, "n": 237, "n_by_type": { "choice": 111, "noul": 57, "score": 69 }, "score": 0.0 } }, "lang": { "ar": { "accuracy": 0.45454545454545453, "cc": -0.15643741530528152, "cc_by_type": { "choice": 20.000000000000004, "noul": -50.0, "score": 29.530687754084152 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 4, "score": 2 }, "score": 0.0 }, "cs": { "accuracy": 0.5, "cc": -12.711864406779661, "cc_by_type": { "choice": 74.57627118644068, "noul": -100.0 }, "low_n": true, "n": 8, "n_by_type": { "choice": 5, "noul": 3 }, "score": 0.0 }, "da": { "accuracy": 0.2916666666666667, "cc": 0.2625190049768242, "cc_by_type": { "choice": 5.325443786982243, "noul": -19.999999999999996, "score": 15.462113227948226 }, "low_n": true, "n": 24, "n_by_type": { "choice": 11, "noul": 5, "score": 8 }, "score": 0.2625190049768242 }, "de": { "accuracy": 0.4222222222222222, "cc": 6.827684779744186, "cc_by_type": { "choice": 29.390464695232346, "noul": -11.111111111111116, "score": 2.2037007551113263 }, "low_n": false, "n": 45, "n_by_type": { "choice": 24, "noul": 9, "score": 12 }, "score": 6.827684779744186 }, "el": { "accuracy": 0.16666666666666666, "cc": -54.072622649773365, "cc_by_type": { "choice": 3.614457831325295, "noul": -100.0, "score": -65.83232578064539 }, "low_n": true, "n": 18, "n_by_type": { "choice": 11, "noul": 5, "score": 2 }, "score": 0.0 }, "en": { "accuracy": 0.3735478105451296, "cc": -4.109282546744619, "cc_by_type": { "choice": 25.29444756029158, "noul": -49.62406015037594, "score": 12.001764949850502 }, "low_n": false, "n": 1119, "n_by_type": { "choice": 577, "noul": 266, "score": 276 }, "score": 0.0 }, "es": { "accuracy": 0.6071428571428571, "cc": 34.262221513293575, "cc_by_type": { "choice": 86.90909090909093, "noul": 9.090909090909083, "score": 6.786664539880705 }, "low_n": true, "n": 28, "n_by_type": { "choice": 10, "noul": 11, "score": 7 }, "score": 34.262221513293575 }, "fi": { "accuracy": 0.3333333333333333, "cc": -31.690140845070424, "cc_by_type": { "choice": 36.61971830985915, "noul": -100.0 }, "low_n": true, "n": 9, "n_by_type": { "choice": 6, "noul": 3 }, "score": 0.0 }, "fr": { "accuracy": 0.4, "cc": -5.790035284708807, "cc_by_type": { "choice": 49.22425952045134, "noul": -100.0, "score": 33.405634625422245 }, "low_n": true, "n": 15, "n_by_type": { "choice": 5, "noul": 4, "score": 6 }, "score": 0.0 }, "hi": { "accuracy": 0.06666666666666667, "cc": -36.64150500544096, "cc_by_type": { "choice": -24.22360248447205, "noul": -100.0, "score": 14.299087468149175 }, "low_n": false, "n": 30, "n_by_type": { "choice": 11, "noul": 10, "score": 9 }, "score": 0.0 }, "id": { "accuracy": 0.3333333333333333, "cc": -32.25806451612903, "cc_by_type": { "choice": 35.483870967741936, "noul": -100.0 }, "low_n": true, "n": 6, "n_by_type": { "choice": 4, "noul": 2 }, "score": 0.0 }, "it": { "accuracy": 0.7647058823529411, "cc": 55.395640882380825, "cc_by_type": { "choice": 49.22425952045134, "noul": 50.0, "score": 66.96266312669114 }, "low_n": true, "n": 17, "n_by_type": { "choice": 5, "noul": 4, "score": 8 }, "score": 55.395640882380825 }, "ja": { "accuracy": 0.21875, "cc": -28.34012514670054, "cc_by_type": { "choice": 10.569105691056912, "noul": -100.0, "score": 4.410518868841473 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 0.0 }, "ko": { "accuracy": 0.2727272727272727, "cc": -19.50727275296486, "cc_by_type": { "choice": 34.42622950819673, "noul": -100.0, "score": 7.0519522329086985 }, "low_n": true, "n": 11, "n_by_type": { "choice": 4, "noul": 2, "score": 5 }, "score": 0.0 }, "mixed": { "accuracy": 0.6363636363636364, "cc": 40.89245662958265, "cc_by_type": { "choice": 78.02197802197803, "noul": 0.0, "score": 44.655391866769904 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 4, "score": 1 }, "score": 40.89245662958265 }, "nl": { "accuracy": 0.23076923076923078, "cc": -23.738139726914895, "cc_by_type": { "choice": -0.2785515320334245, "noul": -33.333333333333336, "score": -37.60253431537792 }, "low_n": true, "n": 13, "n_by_type": { "choice": 5, "noul": 6, "score": 2 }, "score": 0.0 }, "no": { "accuracy": 0.14285714285714285, "cc": -49.18032786885246, "cc_by_type": { "choice": 1.6393442622950836, "noul": -100.0 }, "low_n": true, "n": 7, "n_by_type": { "choice": 4, "noul": 3 }, "score": 0.0 }, "pl": { "accuracy": 0.45, "cc": 25.233491597647614, "cc_by_type": { "choice": 60.64713598600787, "noul": 0.0, "score": 15.05333880693498 }, "low_n": false, "n": 40, "n_by_type": { "choice": 16, "noul": 10, "score": 14 }, "score": 25.233491597647614 }, "pt": { "accuracy": 0.7222222222222222, "cc": 59.348158787470005, "cc_by_type": { "choice": 100.0, "noul": 33.33333333333333, "score": 44.711143029076695 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 6, "score": 5 }, "score": 59.348158787470005 }, "sv": { "accuracy": 0.0, "cc": -47.36188981647681, "cc_by_type": { "choice": -33.333333333333336, "noul": -100.0, "score": -8.752336116097092 }, "low_n": true, "n": 8, "n_by_type": { "choice": 3, "noul": 4, "score": 1 }, "score": 0.0 }, "tr": { "accuracy": 0.45454545454545453, "cc": 0.2881809678820299, "cc_by_type": { "choice": 21.052631578947373, "noul": -33.333333333333336, "score": 13.145244658032052 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 3, "score": 3 }, "score": 0.2881809678820299 }, "uk": { "accuracy": 0.45454545454545453, "cc": -12.288605896536685, "cc_by_type": { "choice": 13.043478260869565, "noul": -100.0, "score": 50.09070404952038 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 1, "score": 4 }, "score": 0.0 }, "zh": { "accuracy": 0.375, "cc": 10.800875659098933, "cc_by_type": { "choice": 2.702702702702704, "noul": 33.33333333333333, "score": -3.6334090587392343 }, "low_n": true, "n": 8, "n_by_type": { "choice": 4, "noul": 3, "score": 1 }, "score": 10.800875659098933 } }, "type": { "choice": { "accuracy": 0.43333333333333335, "cc": 26.58490240110555, "cc_by_type": { "choice": 26.58490240110555 }, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 26.58490240110555 }, "noul": { "accuracy": 0.264, "cc": -47.199999999999996, "cc_by_type": { "noul": -47.199999999999996 }, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.37333333333333335, "cc": 13.164164091005937, "cc_by_type": { "score": 13.164164091005937 }, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 13.164164091005937 } }, "usecase": { "advertising": { "accuracy": 0.375, "cc": -24.495021091479135, "cc_by_type": { "choice": 30.807248764415156, "noul": -100.0, "score": -4.292312038852564 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 0.0 }, "code_linting": { "accuracy": 0.2916666666666667, "cc": -33.15167050235345, "cc_by_type": { "choice": 21.440261865793783, "noul": -100.0, "score": -20.895273372854128 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 0.0 }, "customer_support": { "accuracy": 0.3, "cc": -1.166880782233278, "cc_by_type": { "choice": 35.483870967741936, "noul": -75.0, "score": 36.01548668555823 }, "low_n": true, "n": 20, "n_by_type": { "choice": 8, "noul": 8, "score": 4 }, "score": 0.0 }, "demand_forecasting": { "accuracy": 0.2692307692307692, "cc": -29.51868765932299, "cc_by_type": { "choice": 15.662650602409636, "noul": -100.0, "score": -4.218713580378597 }, "low_n": true, "n": 26, "n_by_type": { "choice": 11, "noul": 4, "score": 11 }, "score": 0.0 }, "ecommerce_marketplaces": { "accuracy": 0.2777777777777778, "cc": -19.515561208758683, "cc_by_type": { "choice": 44.099378881987576, "noul": -100.0, "score": -2.6460625082636247 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 5, "score": 6 }, "score": 0.0 }, "feature_extraction": { "accuracy": 0.2413793103448276, "cc": -22.650212849237985, "cc_by_type": { "choice": 5.9829059829059785, "noul": -60.0, "score": -13.933544530619923 }, "low_n": true, "n": 29, "n_by_type": { "choice": 15, "noul": 5, "score": 9 }, "score": 0.0 }, "financial_crime": { "accuracy": 0.34615384615384615, "cc": -8.626664926886974, "cc_by_type": { "choice": 34.78260869565218, "noul": -75.0, "score": 14.337396523686897 }, "low_n": true, "n": 26, "n_by_type": { "choice": 10, "noul": 8, "score": 8 }, "score": 0.0 }, "gaming": { "accuracy": 0.20689655172413793, "cc": -43.06325104574836, "cc_by_type": { "choice": 4.950495049504946, "noul": -100.0, "score": -34.140248186750014 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 7, "score": 11 }, "score": 0.0 }, "insurance_claims": { "accuracy": 0.42857142857142855, "cc": 16.29704619319791, "cc_by_type": { "choice": 52.38095238095238, "noul": -100.0, "score": 96.51018619864135 }, "low_n": true, "n": 14, "n_by_type": { "choice": 8, "noul": 5, "score": 1 }, "score": 16.29704619319791 }, "knowledge_graphs": { "accuracy": 0.13043478260869565, "cc": -36.43239114303143, "cc_by_type": { "choice": -0.7633587786259586, "noul": -100.0, "score": -8.53381465046834 }, "low_n": true, "n": 23, "n_by_type": { "choice": 14, "noul": 5, "score": 4 }, "score": 0.0 }, "lead_generation": { "accuracy": 0.24, "cc": -15.40910199821853, "cc_by_type": { "choice": 34.959349593495936, "noul": -75.0, "score": -6.186655588151524 }, "low_n": true, "n": 25, "n_by_type": { "choice": 8, "noul": 8, "score": 9 }, "score": 0.0 }, "legal_compliance": { "accuracy": 0.38461538461538464, "cc": 48.23720641132996, "cc_by_type": { "choice": 18.032786885245905, "noul": 100.0, "score": 26.678832348743953 }, "low_n": true, "n": 13, "n_by_type": { "choice": 8, "noul": 1, "score": 4 }, "score": 48.23720641132996 }, "llm_guardrails": { "accuracy": 0.3103448275862069, "cc": -8.538148364008457, "cc_by_type": { "choice": 44.95412844036697, "noul": -75.0, "score": 4.431426467607658 }, "low_n": true, "n": 29, "n_by_type": { "choice": 14, "noul": 8, "score": 7 }, "score": 0.0 }, "model_routing": { "accuracy": 0.5133333333333333, "cc": 32.82106671729687, "cc_by_type": { "choice": 42.106673813990895, "noul": 9.090909090909083, "score": 47.265617246990644 }, "low_n": false, "n": 150, "n_by_type": { "choice": 74, "noul": 44, "score": 32 }, "score": 32.82106671729687 }, "moderation_trust_safety": { "accuracy": 0.1875, "cc": -18.741819385691944, "cc_by_type": { "choice": -10.526315789473685, "noul": -100.0, "score": 54.30085763239786 }, "low_n": true, "n": 16, "n_by_type": { "choice": 8, "noul": 5, "score": 3 }, "score": 0.0 }, "recruiting": { "accuracy": 0.23809523809523808, "cc": -18.91047217979512, "cc_by_type": { "choice": 9.547738693467341, "noul": -100.0, "score": 33.7208447671473 }, "low_n": true, "n": 21, "n_by_type": { "choice": 9, "noul": 6, "score": 6 }, "score": 0.0 }, "risk_assessment": { "accuracy": 0.46153846153846156, "cc": 6.905355691048825, "cc_by_type": { "choice": 43.39622641509434, "noul": 33.33333333333333, "score": -56.0134926752812 }, "low_n": true, "n": 13, "n_by_type": { "choice": 7, "noul": 3, "score": 3 }, "score": 6.905355691048825 }, "scientific_discovery": { "accuracy": 0.25, "cc": -23.30406902667455, "cc_by_type": { "choice": 20.31872509960159, "noul": -100.0, "score": 9.769067820374755 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 0.0 }, "search_retrieval": { "accuracy": 0.2727272727272727, "cc": -24.90976631788278, "cc_by_type": { "choice": 28.40095465393794, "noul": -100.0, "score": -3.1302536075862752 }, "low_n": true, "n": 22, "n_by_type": { "choice": 9, "noul": 6, "score": 7 }, "score": 0.0 } } }, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 72.49837033035136, 80.76387849103644 ], "I": [ 12.343340329004636, 18.854234371958732 ], "capability": [ 43.298750810210294, 49.02128383708443 ], "composite": { "A": [ 2.071813842220017, 6.329337632277144 ], "B": [ 1.533726731769256, 4.973696434425022 ], "C": [ 1.4387596126527897, 4.395373355748017 ] }, "seed": 22 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": false, "g_med_used": 10.073244581339713, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 78.54210067220151, "capability": 47.20710280189251, "intelligence": 15.872104931583502 }, "per_type": { "choice": { "accuracy": 0.43333333333333335, "calibration": 71.36245954848286, "cc": 26.58490240110555, "cc_by_type": { "choice": 26.58490240110555 }, "cc_tiered": 29.996434046841067, "cc_tiered_open": 35.509112308155075, "cc_tiered_sealed": 24.483755785527055, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 26.58490240110555 }, "noul": { "accuracy": 0.264, "calibration": 77.24881325795822, "cc": -47.199999999999996, "cc_by_type": { "noul": -47.199999999999996 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.37333333333333335, "calibration": 87.01502921016349, "cc": 13.164164091005937, "cc_by_type": { "score": 13.164164091005937 }, "cc_tiered": 17.61988074790945, "cc_tiered_open": 26.09045715280502, "cc_tiered_sealed": 9.149304343013874, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 13.164164091005937 } }, "raw_on_A": { "calibration": 73.78928446595333, "capability": 44.008100936829386, "intelligence": 14.22691740770544 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.014666666666666717, "n": 750 }, "noul": { "invalid_rate": 0.029333333333333322, "n": 375 }, "score": { "invalid_rate": 0.01866666666666672, "n": 375 } }, "views": { "Balanced 33:33:33": 3.5006149102123802, "Emphasis on Accuracy 60:20:20": 2.372543895109552, "Emphasis on Cost 20:20:60": 4.503957993600416, "Emphasis on Speed 20:60:20": 4.684348850018089, "Intelligence only": 1.599423855434685 } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "eljiwo / jiwidi (fhjaime96@gmail.com)", "axes": { "calibration": 85.66971124396188, "cost": 64.0899740452639, "intelligence": 34.784391844693545, "speed": 92.44547609658223 }, "calibration": { "parts": { "choice": { "ece_hard": 0.0747562710407059, "ece_score": 85.04874579185882, "mean_tvd": null, "n": 739, "n_ece_hard": 317, "n_tvd": 0, "score": 85.04874579185882 }, "noul": { "abstention_rate": 0.37362637362637363, "brier": 0.1479946142436308, "ece": 0.06661725466202273, "n": 364, "score": 86.67654906759546 }, "score": { "interval90_coverage": 0.9402173913043478, "interval90_width": 0.48007246376811585, "n": 368, "nrps": 0.1073762318235206, "score": 85.28383887243137, "top_ece": 0.09347349536392596 } }, "score": 85.66971124396188 }, "capability": 60.227051544327715, "composite_ci95": { "A": [ 18.62937312636143, 39.08706741447612 ], "B": [ 15.795090114565683, 34.93479875570867 ], "C": [ 12.937064671084322, 27.143796815608415 ] }, "cost": { "basis": "ESTIMATE (*est.), exact base-model market reference; self-hosted open weights with no per-token tariff of their own. The unmodified frozen pricing_v15 resolves the author's own declared base spelling Qwen/Qwen3.5-4B straight through to deepinfra:Qwen/Qwen3.5-4B at USD 0.03 per 1M input and USD 0.15 per 1M output in the dated 25 Sep 2026 snapshot. No new release-lane pricing line.; reference `deepinfra:Qwen/Qwen3.5-4B` frozen at the 25 Sep cut-off, deprecated afterwards (replaced by `Qwen/Qwen3.5-9B`); re-score if it changes (I-3) OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over the item's own candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone. This row's own usage on draw v1.6-regular-20261010-a2: 786,993 input and 0 output tokens over 1,500 decisions. Estimated, not charged.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01573986 }, "display": "jiwo 4b (Qwen3.5-4B backbone with a trained decision head, one forward pass, native probabilities, self-hosted)", "endpoint_condition": "The author's own install line from their model card (pip install 'jiwo[cuda] @ git+https://github.com/jiwidi/jiwo@8716abab') and their own `jiwo serve`, nothing patched. The server reads JIWO_CHECKPOINT and calls from_pretrained with NO revision and there is no JIWO_REVISION setting, so it was pointed at the LOCAL PINNED SNAPSHOT DIRECTORY and the served revision cannot have drifted to main. JIWO_MAX_LENGTH=65536, the author's own documented value for their published runs: the 8,192 default refuses longer prompts with HTTP 400 and never truncates. The trained context is 8,192 tokens, so the longest items of this draw are answered outside the range the model saw in training. CUDA graphs are OFF (JIWO_CUDA_GRAPHS unset): the author's own card says graphs agree with eager on only 99.7 % of fields, so eager is the exact path and the author's 14.5 ms headline uses a mode this measurement does not use. Requests are serialised by the author's own asyncio.Lock. evaluator-owned Lium RTX 5090 32 GB pod (pod calm-wolf-31, USD 0.72/h; Brasov, Romania, EU by the RIPE whois of its public IP, receipt r90/receipts/POD-IP-WHOIS-R90.txt); pod-level iptables egress cut installed, verified and TIMESTAMPED pod-side (logs/EGRESS-CUT-R90.log) BEFORE the sealed input was uploaded, so the ordering is provable from pod evidence and not only from the session transcript; HF/transformers offline, no credentials in the environment, one decision per request, no batching, no retry. The five rows ran strictly one at a time on the one card, each server started with setsid and stopped by its exact process group with a card-free check in between.", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 41.202737997549946, "I_sealed": 28.366045691837137, "base": 34.784391844693545, "excess": 2.763447724373096, "gap": 12.836692305712809, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 61.6558109869045, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864866, "hard": 58.47750865051904, "judge": 60.44825313117996, "standard": 62.32733361239012 } }, "open|noul": { "cc": 26.298826777087655, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": 21.739130434782616, "judge": 10.000000000000009, "standard": 44.44444444444444 } }, "open|score": { "cc": 35.653576228657684, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 50.10967244172717, "hard": 38.322559765790174, "judge": 32.507251953307225, "standard": 27.80704746088366 } }, "sealed|choice": { "cc": 51.626839698704124, "n": { "easy": 72, "hard": 260, "judge": 141, "standard": 127 }, "tiers": { "easy": 64.33413286056941, "hard": 48.059799934044186, "judge": 51.60062909035564, "standard": 52.446588559614064 } }, "sealed|noul": { "cc": 4.118819776714509, "n": { "easy": 39, "hard": 132, "judge": 72, "standard": 57 }, "tiers": { "easy": 33.33333333333333, "hard": -7.57575757575758, "judge": 13.888888888888884, "standard": -1.7543859649122862 } }, "sealed|score": { "cc": 29.352477600092772, "n": { "easy": 33, "hard": 115, "judge": 75, "standard": 77 }, "tiers": { "easy": 53.37012049696441, "hard": 14.78666626313042, "judge": 38.03240184856901, "standard": 33.455392452867294 } } } }, "jevbench_score": 28.962932355698086, "key": "jiwo-4b", "last_measured_on": "2026-10-10", "licence": "Apache-2.0 (weights, model card); MIT (code, github.com/jiwidi/jiwo)", "listing": "ranked", "measured_in": "v1.6.9", "measurement_date_status": "measured", "model_pin": "eljiwo/jiwo-4b@9adc559ff7ec5baffade3998f7bca85da0d124ed", "noul_decisive": { "acc_among_decisive": 0.8991228070175439, "decisive": 228, "decisive_rate": 0.6263736263736264, "n": 375, "supported": true, "valid": 364 }, "planned_weight_denominator": 1.0, "rank": 3, "ranked": true, "ranks": { "A": 3, "B": 3, "C": 4, "capability": 4 }, "repo": "https://huggingface.co/eljiwo/jiwo-4b", "scores": { "A": 28.962932355698086, "B": 25.315480694857687, "C": 20.113147469234782 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.21877912301570177, "p50_s_raw": 0.034389561507850885, "p95_s_adjusted": 0.26028342521749437, "p95_s_raw": 0.05514171260874719 }, "status": { "answered_ok": 1471, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns": { "family": { "calibration": { "accuracy": 0.7586206896551724, "cc": 63.41069040078068, "cc_by_type": { "choice": 66.66666666666667, "noul": 66.66666666666667, "score": 56.8987378690087 }, "low_n": true, "n": 29, "n_by_type": { "choice": 16, "noul": 6, "score": 7 }, "score": 63.41069040078068 }, "calibration_gen": { "accuracy": 0.5446428571428571, "cc": 21.306873408130947, "cc_by_type": { "choice": 51.08695652173913, "noul": 0.0, "score": 12.833663702653709 }, "low_n": false, "n": 112, "n_by_type": { "choice": 56, "noul": 26, "score": 30 }, "score": 21.306873408130947 }, "composite_gen": { "accuracy": 0.6020408163265306, "cc": 40.276766724193266, "cc_by_type": { "choice": 62.96296296296296, "noul": 21.42857142857142, "score": 36.438765781045426 }, "low_n": false, "n": 98, "n_by_type": { "choice": 36, "noul": 28, "score": 34 }, "score": 40.276766724193266 }, "fairness": { "accuracy": 1.0, "cc": 97.09800473660214, "cc_by_type": { "choice": 100.0, "noul": 100.0, "score": 91.29401420980636 }, "low_n": true, "n": 27, "n_by_type": { "choice": 12, "noul": 6, "score": 9 }, "score": 97.09800473660214 }, "jaggedness": { "accuracy": 0.372972972972973, "cc": -11.082947542546755, "cc_by_type": { "choice": 29.133858267716537, "noul": -69.23076923076923, "score": 6.848068335412427 }, "low_n": false, "n": 185, "n_by_type": { "choice": 97, "noul": 39, "score": 49 }, "score": 0.0 }, "llm_router": { "accuracy": 0.6893939393939394, "cc": 58.2269224408714, "cc_by_type": { "choice": 51.69530887134232, "noul": 50.68493150684932, "score": 72.30052694442254 }, "low_n": false, "n": 264, "n_by_type": { "choice": 131, "noul": 73, "score": 60 }, "score": 58.2269224408714 }, "long_state": { "accuracy": 0.0, "cc": -110.31746031746032, "cc_by_type": { "choice": -30.952380952380945, "noul": -100.0, "score": -200.0 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 11, "score": 7 }, "score": 0.0 }, "ml_usecase": { "accuracy": 0.3870967741935484, "cc": -16.440841671706764, "cc_by_type": { "choice": 16.923076923076923, "noul": -75.0, "score": 8.754398061802782 }, "low_n": false, "n": 31, "n_by_type": { "choice": 15, "noul": 8, "score": 8 }, "score": 0.0 }, "multilingual": { "accuracy": 0.6158730158730159, "cc": 34.72665534729893, "cc_by_type": { "choice": 57.00850873264667, "noul": 17.24137931034482, "score": 29.930077998905315 }, "low_n": false, "n": 315, "n_by_type": { "choice": 146, "noul": 87, "score": 82 }, "score": 34.72665534729893 }, "negation_pair_gen": { "accuracy": 0.5892857142857143, "cc": 2.0857022253018394, "cc_by_type": { "choice": 57.142857142857146, "noul": 10.000000000000009, "score": -60.885750466951635 }, "low_n": false, "n": 56, "n_by_type": { "choice": 28, "noul": 20, "score": 8 }, "score": 2.0857022253018394 }, "option_order": { "accuracy": 0.9122807017543859, "cc": 88.90122086570477, "cc_by_type": { "choice": 88.90122086570477 }, "low_n": false, "n": 57, "n_by_type": { "choice": 57 }, "score": 88.90122086570477 }, "perturb_pair": { "accuracy": 0.85, "cc": 73.88280168114402, "cc_by_type": { "choice": 84.27260812581913, "noul": 57.14285714285714, "score": 80.23293977475579 }, "low_n": false, "n": 60, "n_by_type": { "choice": 34, "noul": 14, "score": 12 }, "score": 73.88280168114402 }, "usecase": { "accuracy": 0.5485232067510548, "cc": 27.448440267073206, "cc_by_type": { "choice": 51.4357294276354, "noul": 1.7543859649122862, "score": 29.155205408671925 }, "low_n": false, "n": 237, "n_by_type": { "choice": 111, "noul": 57, "score": 69 }, "score": 27.448440267073206 } }, "lang": { "ar": { "accuracy": 0.45454545454545453, "cc": -16.233054801263584, "cc_by_type": { "choice": 20.000000000000004, "noul": -50.0, "score": -18.699164403790757 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 4, "score": 2 }, "score": 0.0 }, "cs": { "accuracy": 0.5, "cc": 28.531073446327685, "cc_by_type": { "choice": 23.72881355932204, "noul": 33.33333333333333 }, "low_n": true, "n": 8, "n_by_type": { "choice": 5, "noul": 3 }, "score": 28.531073446327685 }, "da": { "accuracy": 0.5833333333333334, "cc": 40.559189285398894, "cc_by_type": { "choice": 40.8284023668639, "noul": 19.999999999999996, "score": 60.84916548933277 }, "low_n": true, "n": 24, "n_by_type": { "choice": 11, "noul": 5, "score": 8 }, "score": 40.559189285398894 }, "de": { "accuracy": 0.6888888888888889, "cc": 43.45157490330706, "cc_by_type": { "choice": 72.84248642124321, "noul": 33.33333333333333, "score": 24.178904955344628 }, "low_n": false, "n": 45, "n_by_type": { "choice": 24, "noul": 9, "score": 12 }, "score": 43.45157490330706 }, "el": { "accuracy": 0.4444444444444444, "cc": -27.556284563855588, "cc_by_type": { "choice": 39.7590361445783, "noul": -60.0, "score": -62.42788983614507 }, "low_n": true, "n": 18, "n_by_type": { "choice": 11, "noul": 5, "score": 2 }, "score": 0.0 }, "en": { "accuracy": 0.5889186773905273, "cc": 30.320127791507446, "cc_by_type": { "choice": 53.7857543466068, "noul": 5.263157894736836, "score": 31.911471133178704 }, "low_n": false, "n": 1119, "n_by_type": { "choice": 577, "noul": 266, "score": 276 }, "score": 30.320127791507446 }, "es": { "accuracy": 0.75, "cc": 53.31121705049497, "cc_by_type": { "choice": 73.81818181818181, "noul": 45.45454545454546, "score": 40.660923878757615 }, "low_n": true, "n": 28, "n_by_type": { "choice": 10, "noul": 11, "score": 7 }, "score": 53.31121705049497 }, "fi": { "accuracy": 0.4444444444444444, "cc": 24.4131455399061, "cc_by_type": { "choice": 15.49295774647887, "noul": 33.33333333333333 }, "low_n": true, "n": 9, "n_by_type": { "choice": 6, "noul": 3 }, "score": 24.4131455399061 }, "fr": { "accuracy": 0.6, "cc": 19.936766375030352, "cc_by_type": { "choice": 74.61212976022568, "noul": -50.0, "score": 35.19816936486537 }, "low_n": true, "n": 15, "n_by_type": { "choice": 5, "noul": 4, "score": 6 }, "score": 19.936766375030352 }, "hi": { "accuracy": 0.43333333333333335, "cc": 12.211958226018675, "cc_by_type": { "choice": 25.465838509316765, "noul": 0.0, "score": 11.170036168739262 }, "low_n": false, "n": 30, "n_by_type": { "choice": 11, "noul": 10, "score": 9 }, "score": 12.211958226018675 }, "id": { "accuracy": 0.8333333333333334, "cc": 50.0, "cc_by_type": { "choice": 100.0, "noul": 0.0 }, "low_n": true, "n": 6, "n_by_type": { "choice": 4, "noul": 2 }, "score": 50.0 }, "it": { "accuracy": 0.8235294117647058, "cc": 65.35071606453869, "cc_by_type": { "choice": 100.00000000000001, "noul": 50.0, "score": 46.05214819361604 }, "low_n": true, "n": 17, "n_by_type": { "choice": 5, "noul": 4, "score": 8 }, "score": 65.35071606453869 }, "ja": { "accuracy": 0.4375, "cc": 12.03476070227424, "cc_by_type": { "choice": 34.959349593495936, "noul": -14.28571428571429, "score": 15.430646799041082 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 12.03476070227424 }, "ko": { "accuracy": 0.6363636363636364, "cc": 57.974720722270284, "cc_by_type": { "choice": 67.21311475409836, "noul": 100.0, "score": 6.711047412712501 }, "low_n": true, "n": 11, "n_by_type": { "choice": 4, "noul": 2, "score": 5 }, "score": 57.974720722270284 }, "mixed": { "accuracy": 1.0, "cc": 89.40690554679048, "cc_by_type": { "choice": 100.0, "noul": 100.0, "score": 68.22071664037144 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 4, "score": 1 }, "score": 89.40690554679048 }, "nl": { "accuracy": 0.5384615384615384, "cc": 30.176686204383895, "cc_by_type": { "choice": 49.860724233983284, "noul": 0.0, "score": 40.66933437916839 }, "low_n": true, "n": 13, "n_by_type": { "choice": 5, "noul": 6, "score": 2 }, "score": 30.176686204383895 }, "no": { "accuracy": 0.7142857142857143, "cc": 67.21311475409837, "cc_by_type": { "choice": 34.42622950819673, "noul": 100.0 }, "low_n": true, "n": 7, "n_by_type": { "choice": 4, "noul": 3 }, "score": 67.21311475409837 }, "pl": { "accuracy": 0.6, "cc": 33.93999941140274, "cc_by_type": { "choice": 76.38828159160472, "noul": 0.0, "score": 25.431716642603497 }, "low_n": false, "n": 40, "n_by_type": { "choice": 16, "noul": 10, "score": 14 }, "score": 33.93999941140274 }, "pt": { "accuracy": 0.8888888888888888, "cc": 77.91153300960076, "cc_by_type": { "choice": 81.90954773869345, "noul": 66.66666666666667, "score": 85.15838462344217 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 6, "score": 5 }, "score": 77.91153300960076 }, "sv": { "accuracy": 0.375, "cc": -8.765750483399948, "cc_by_type": { "choice": -33.333333333333336, "noul": 50.0, "score": -42.96391811686651 }, "low_n": true, "n": 8, "n_by_type": { "choice": 3, "noul": 4, "score": 1 }, "score": 0.0 }, "tr": { "accuracy": 0.6363636363636364, "cc": 24.254058362470982, "cc_by_type": { "choice": 73.6842105263158, "noul": 33.33333333333333, "score": -34.25536877223618 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 3, "score": 3 }, "score": 24.254058362470982 }, "uk": { "accuracy": 0.45454545454545453, "cc": -18.250772299457207, "cc_by_type": { "choice": 13.043478260869565, "noul": -100.0, "score": 32.20420484075882 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 1, "score": 4 }, "score": 0.0 }, "zh": { "accuracy": 0.75, "cc": 61.21814460262698, "cc_by_type": { "choice": 67.56756756756756, "noul": 33.33333333333333, "score": 82.75353290698004 }, "low_n": true, "n": 8, "n_by_type": { "choice": 4, "noul": 3, "score": 1 }, "score": 61.21814460262698 } }, "type": { "choice": { "accuracy": 0.6453333333333333, "cc": 54.05078597339782, "cc_by_type": { "choice": 54.05078597339782 }, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 54.05078597339782 }, "noul": { "accuracy": 0.5466666666666666, "cc": 9.333333333333327, "cc_by_type": { "noul": 9.333333333333327 }, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 9.333333333333327 }, "score": { "accuracy": 0.5413333333333333, "cc": 31.101047707694065, "cc_by_type": { "score": 31.101047707694065 }, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 31.101047707694065 } }, "usecase": { "advertising": { "accuracy": 0.5833333333333334, "cc": 24.071004328629304, "cc_by_type": { "choice": 40.691927512355846, "noul": -19.999999999999996, "score": 51.52108547353207 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 24.071004328629304 }, "code_linting": { "accuracy": 0.5833333333333334, "cc": 39.264007955407884, "cc_by_type": { "choice": 41.08019639934533, "noul": 60.00000000000001, "score": 16.711827466878294 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 39.264007955407884 }, "customer_support": { "accuracy": 0.5, "cc": 20.00867761558955, "cc_by_type": { "choice": 35.483870967741936, "noul": -25.0, "score": 49.54216187902672 }, "low_n": true, "n": 20, "n_by_type": { "choice": 8, "noul": 8, "score": 4 }, "score": 20.00867761558955 }, "demand_forecasting": { "accuracy": 0.38461538461538464, "cc": -10.818142384890914, "cc_by_type": { "choice": 27.710843373493972, "noul": -50.0, "score": -10.165270528166715 }, "low_n": true, "n": 26, "n_by_type": { "choice": 11, "noul": 4, "score": 11 }, "score": 0.0 }, "ecommerce_marketplaces": { "accuracy": 0.5, "cc": 24.55248819664185, "cc_by_type": { "choice": 62.73291925465839, "noul": -19.999999999999996, "score": 30.92454533526716 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 5, "score": 6 }, "score": 24.55248819664185 }, "feature_extraction": { "accuracy": 0.4482758620689655, "cc": 12.840294527779639, "cc_by_type": { "choice": 23.076923076923077, "noul": -19.999999999999996, "score": 35.44396050641584 }, "low_n": true, "n": 29, "n_by_type": { "choice": 15, "noul": 5, "score": 9 }, "score": 12.840294527779639 }, "financial_crime": { "accuracy": 0.6153846153846154, "cc": 42.181818059125035, "cc_by_type": { "choice": 47.82608695652174, "noul": 75.0, "score": 3.7193672208533535 }, "low_n": true, "n": 26, "n_by_type": { "choice": 10, "noul": 8, "score": 8 }, "score": 42.181818059125035 }, "gaming": { "accuracy": 0.5172413793103449, "cc": 15.436535058528824, "cc_by_type": { "choice": 52.475247524752476, "noul": -14.28571428571429, "score": 8.120071936548289 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 7, "score": 11 }, "score": 15.436535058528824 }, "insurance_claims": { "accuracy": 0.5, "cc": 2.555674036706544, "cc_by_type": { "choice": 68.25396825396825, "noul": -60.0, "score": -0.5869461438486212 }, "low_n": true, "n": 14, "n_by_type": { "choice": 8, "noul": 5, "score": 1 }, "score": 2.555674036706544 }, "knowledge_graphs": { "accuracy": 0.4782608695652174, "cc": 14.125622525737157, "cc_by_type": { "choice": 35.87786259541985, "noul": -19.999999999999996, "score": 26.499004981791618 }, "low_n": true, "n": 23, "n_by_type": { "choice": 14, "noul": 5, "score": 4 }, "score": 14.125622525737157 }, "lead_generation": { "accuracy": 0.4, "cc": 11.047439957865064, "cc_by_type": { "choice": 83.73983739837398, "noul": -50.0, "score": -0.59751752477879 }, "low_n": true, "n": 25, "n_by_type": { "choice": 8, "noul": 8, "score": 9 }, "score": 11.047439957865064 }, "legal_compliance": { "accuracy": 0.46153846153846156, "cc": -11.127349456353167, "cc_by_type": { "choice": 34.42622950819673, "noul": -100.0, "score": 32.19172212274377 }, "low_n": true, "n": 13, "n_by_type": { "choice": 8, "noul": 1, "score": 4 }, "score": 0.0 }, "llm_guardrails": { "accuracy": 0.6551724137931034, "cc": 53.749288065206805, "cc_by_type": { "choice": 63.30275229357799, "noul": 50.0, "score": 47.945111902042434 }, "low_n": true, "n": 29, "n_by_type": { "choice": 14, "noul": 8, "score": 7 }, "score": 53.749288065206805 }, "model_routing": { "accuracy": 0.7133333333333334, "cc": 58.29415940370154, "cc_by_type": { "choice": 64.62074510855, "noul": 59.09090909090908, "score": 51.17082401164556 }, "low_n": false, "n": 150, "n_by_type": { "choice": 74, "noul": 44, "score": 32 }, "score": 58.29415940370154 }, "moderation_trust_safety": { "accuracy": 0.5, "cc": 16.05980603069598, "cc_by_type": { "choice": 52.63157894736842, "noul": -60.0, "score": 55.547839144719525 }, "low_n": true, "n": 16, "n_by_type": { "choice": 8, "noul": 5, "score": 3 }, "score": 16.05980603069598 }, "recruiting": { "accuracy": 0.6190476190476191, "cc": 33.863204600776136, "cc_by_type": { "choice": 69.84924623115576, "noul": 0.0, "score": 31.740367571172644 }, "low_n": true, "n": 21, "n_by_type": { "choice": 9, "noul": 6, "score": 6 }, "score": 33.863204600776136 }, "risk_assessment": { "accuracy": 0.3076923076923077, "cc": -16.123855817090945, "cc_by_type": { "choice": 5.660377358490565, "noul": -33.333333333333336, "score": -20.698611476430063 }, "low_n": true, "n": 13, "n_by_type": { "choice": 7, "noul": 3, "score": 3 }, "score": 0.0 }, "scientific_discovery": { "accuracy": 0.375, "cc": -7.830166742245569, "cc_by_type": { "choice": 28.286852589641434, "noul": -71.42857142857143, "score": 19.65121861219329 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 0.0 }, "search_retrieval": { "accuracy": 0.8181818181818182, "cc": 66.53707551936073, "cc_by_type": { "choice": 85.68019093078759, "noul": 66.66666666666667, "score": 47.26436896062795 }, "low_n": true, "n": 22, "n_by_type": { "choice": 9, "noul": 6, "score": 7 }, "score": 66.53707551936073 } } }, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 80.08166368366025, 86.75710341088941 ], "I": [ 29.097201235740425, 39.427338008588556 ], "capability": [ 55.64492730946361, 62.239450216352004 ], "composite": { "A": [ 18.62937312636143, 39.08706741447612 ], "B": [ 15.795090114565683, 34.93479875570867 ], "C": [ 12.937064671084322, 27.143796815608415 ] }, "seed": 23 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": false, "g_med_used": 10.073244581339713, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 85.66971124396188, "capability": 60.227051544327715, "intelligence": 34.784391844693545 }, "per_type": { "choice": { "accuracy": 0.6453333333333333, "calibration": 85.04874579185882, "cc": 54.05078597339782, "cc_by_type": { "choice": 54.05078597339782 }, "cc_tiered": 56.64132534280431, "cc_tiered_open": 61.6558109869045, "cc_tiered_sealed": 51.626839698704124, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 54.05078597339782 }, "noul": { "accuracy": 0.5466666666666666, "calibration": 86.67654906759546, "cc": 9.333333333333327, "cc_by_type": { "noul": 9.333333333333327 }, "cc_tiered": 15.208823276901082, "cc_tiered_open": 26.298826777087655, "cc_tiered_sealed": 4.118819776714509, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 9.333333333333327 }, "score": { "accuracy": 0.5413333333333333, "calibration": 85.28383887243137, "cc": 31.101047707694065, "cc_by_type": { "score": 31.101047707694065 }, "cc_tiered": 32.503026914375226, "cc_tiered_open": 35.653576228657684, "cc_tiered_sealed": 29.352477600092772, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 31.101047707694065 } }, "raw_on_A": { "calibration": 82.12950185196465, "capability": 58.61658974199523, "intelligence": 35.1036776320258 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.014666666666666717, "n": 750 }, "noul": { "invalid_rate": 0.029333333333333322, "n": 375 }, "score": { "invalid_rate": 0.01866666666666672, "n": 375 } }, "views": { "Balanced 33:33:33": 26.318232161775516, "Emphasis on Accuracy 60:20:20": 21.478631369812923, "Emphasis on Cost 20:20:60": 28.0163293235414, "Emphasis on Speed 20:60:20": 31.50792357388672, "Intelligence only": 16.835004451338282 } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Liquid AI", "axes": { "calibration": 76.87411794897184, "cost": 50.56108621702079, "intelligence": 22.13919485880221, "speed": 94.32773191369529 }, "calibration": { "parts": { "choice": { "ece_hard": 0.11313979352292604, "ece_score": 77.37204129541479, "mean_tvd": null, "n": 750, "n_ece_hard": 328, "n_tvd": 0, "score": 77.37204129541479 }, "noul": { "abstention_rate": 0.5093333333333334, "brier": 0.19849004201230716, "ece": 0.11360395591990471, "n": 375, "score": 77.27920881601906 }, "score": { "interval90_coverage": 0.9093333333333333, "interval90_width": 0.5644444444444446, "n": 375, "nrps": 0.1509830793884296, "score": 75.9711037354817, "top_ece": 0.16479742295096816 } }, "score": 76.87411794897184 }, "capability": 49.50665640388703, "composite_ci95": { "A": [ 5.218755487589073, 12.205081733672536 ], "B": [ 4.193052691245517, 10.348794476091657 ], "C": [ 3.6241357552701894, 8.475751203939263 ] }, "cost": { "basis": "ESTIMATE (*est.), size-class reference; self-hosted open weights, no bookable listing for this base. The declared base LiquidAI/LFM2.5-VL-3B is not a key of any frozen pricing table, and the frozen 25 Sep 2026 snapshot's only Liquid entry is liquid/lfm-2.5-2.6b:free at (0, 0) - a free route, which may not price a row (Florian, free tiers: use the published paid price or the estimate). Method, as aplomb-1, wald-4b-v2, jiwo-* and the ClassOne trio carry: the DeepInfra Qwen3.5 size-class reference from the frozen snapshot, rounded ONE CLASS UP so the estimate can only OVERSTATE this row's cost - deepinfra:Qwen/Qwen3.5-4B at USD 0.03 per 1M input and USD 0.15 per 1M output, for a 3B model. Applied to this row's own measured usage: 2,222,971 input tokens and 0 output tokens over 1,500 decisions. The 2B rate (0.02, 0.10) would give 0.029639613333333335 and was deliberately NOT used because it understates. Precedent for a LiquidAI base priced as a labelled estimate: the live lev-350m row (LiquidAI/LFM2.5-350M, in UNLISTED_BASE_MODELS, estimate 0.00462807881773399 per 1,000).", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.044459420000000006 }, "display": "Liquid AI d1-3B", "endpoint_condition": "evaluator-owned Lium RTX 5090 32 GB pod (France), offline after setup (egress cut and proven), author's custom code at the pinned commit via trust_remote_code from the sha256-verified local copy, bfloat16, no torch.compile", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 27.85582362507926, "I_sealed": 16.42256609252516, "base": 22.13919485880221, "excess": 1.3600129512143884, "gap": 11.433257532554101, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 53.72008003683186, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675676, "hard": 43.378420887071414, "judge": 60.44825313117996, "standard": 54.79280033486815 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": -30.434782608695656, "judge": -19.999999999999996, "standard": -11.111111111111116 } }, "open|score": { "cc": 29.847390838405925, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 36.55757331890299, "hard": 17.69560694707587, "judge": 41.70520484649972, "standard": 33.00914636867683 } }, "sealed|choice": { "cc": 39.056033872257295, "n": { "easy": 72, "hard": 260, "judge": 141, "standard": 127 }, "tiers": { "easy": 54.948378350192925, "hard": 33.21974277234252, "judge": 38.815889604789206, "standard": 43.14266023432116 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 39, "hard": 132, "judge": 72, "standard": 57 }, "tiers": { "easy": 7.692307692307687, "hard": -45.45454545454546, "judge": -5.555555555555558, "standard": -5.263157894736848 } }, "sealed|score": { "cc": 10.211664405318189, "n": { "easy": 33, "hard": 115, "judge": 75, "standard": 77 }, "tiers": { "easy": 27.92356315993724, "hard": 9.442062390485162, "judge": 0.13958879484807785, "standard": 18.003032473379875 } } } }, "jevbench_score": 8.855712627780655, "key": "liquid-d1-3b", "last_measured_on": "2026-10-10", "licence": "LFM Open License v1.0 (commercial use only below USD 10 M annual revenue)", "listing": "ranked", "measured_in": "v1.6.5", "measurement_date_status": "measured", "model_pin": "LiquidAI/d1-3B@051bcc464b01b9f92942b364d9586b0ef5912432", "noul_decisive": { "acc_among_decisive": 0.8315217391304348, "decisive": 184, "decisive_rate": 0.49066666666666664, "n": 375, "supported": true, "valid": 375 }, "planned_weight_denominator": 1.0, "rank": 8, "ranked": true, "ranks": { "A": 8, "B": 8, "C": 8, "capability": 7 }, "repo": "https://huggingface.co/LiquidAI/d1-3B", "scores": { "A": 8.855712627780655, "B": 7.330614913163864, "C": 6.149800435958788 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.1797857349505648, "p50_s_raw": 0.0148928674752824, "p95_s_adjusted": 0.20533906437922267, "p95_s_raw": 0.027669532189611334 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns": { "family": { "calibration": { "accuracy": 0.7241379310344828, "cc": 54.390853941466354, "cc_by_type": { "choice": 75.0, "noul": 66.66666666666667, "score": 21.50589515773239 }, "low_n": true, "n": 29, "n_by_type": { "choice": 16, "noul": 6, "score": 7 }, "score": 54.390853941466354 }, "calibration_gen": { "accuracy": 0.45535714285714285, "cc": 7.885435170467154, "cc_by_type": { "choice": 41.770186335403736, "noul": 0.0, "score": -18.113880824002273 }, "low_n": false, "n": 112, "n_by_type": { "choice": 56, "noul": 26, "score": 30 }, "score": 7.885435170467154 }, "composite_gen": { "accuracy": 0.5408163265306123, "cc": 25.303042500575433, "cc_by_type": { "choice": 77.77777777777779, "noul": -21.42857142857143, "score": 19.559921152519944 }, "low_n": false, "n": 98, "n_by_type": { "choice": 36, "noul": 28, "score": 34 }, "score": 25.303042500575433 }, "fairness": { "accuracy": 0.8518518518518519, "cc": 47.39465841291355, "cc_by_type": { "choice": 88.23529411764707, "noul": 0.0, "score": 53.948681121093585 }, "low_n": true, "n": 27, "n_by_type": { "choice": 12, "noul": 6, "score": 9 }, "score": 47.39465841291355 }, "jaggedness": { "accuracy": 0.31891891891891894, "cc": -22.274181763566684, "cc_by_type": { "choice": 20.79666512274201, "noul": -79.48717948717949, "score": -8.13203092626258 }, "low_n": false, "n": 185, "n_by_type": { "choice": 97, "noul": 39, "score": 49 }, "score": 0.0 }, "llm_router": { "accuracy": 0.625, "cc": 51.3034512100294, "cc_by_type": { "choice": 47.05062703204832, "noul": 50.68493150684932, "score": 56.17479509119057 }, "low_n": false, "n": 264, "n_by_type": { "choice": 131, "noul": 73, "score": 60 }, "score": 51.3034512100294 }, "long_state": { "accuracy": 0.1724137931034483, "cc": -29.57896952664178, "cc_by_type": { "choice": 16.66666666666667, "noul": -100.0, "score": -5.403575246592007 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 11, "score": 7 }, "score": 0.0 }, "ml_usecase": { "accuracy": 0.2903225806451613, "cc": -13.862612846690979, "cc_by_type": { "choice": 7.692307692307688, "noul": -50.0, "score": 0.7198537676193717 }, "low_n": false, "n": 31, "n_by_type": { "choice": 15, "noul": 8, "score": 8 }, "score": 0.0 }, "multilingual": { "accuracy": 0.41904761904761906, "cc": 4.0623722132052436, "cc_by_type": { "choice": 34.617107030900144, "noul": -33.333333333333336, "score": 10.903342942048921 }, "low_n": false, "n": 315, "n_by_type": { "choice": 146, "noul": 87, "score": 82 }, "score": 4.0623722132052436 }, "negation_pair_gen": { "accuracy": 0.44642857142857145, "cc": -3.4722633613639875, "cc_by_type": { "choice": 38.095238095238095, "noul": -50.0, "score": 1.4879718206699422 }, "low_n": false, "n": 56, "n_by_type": { "choice": 28, "noul": 20, "score": 8 }, "score": 0.0 }, "option_order": { "accuracy": 0.7017543859649122, "cc": 62.264150943396224, "cc_by_type": { "choice": 62.264150943396224 }, "low_n": false, "n": 57, "n_by_type": { "choice": 57 }, "score": 62.264150943396224 }, "perturb_pair": { "accuracy": 0.7166666666666667, "cc": 57.31023460546714, "cc_by_type": { "choice": 52.81782437745741, "noul": 42.85714285714286, "score": 76.25573658180112 }, "low_n": false, "n": 60, "n_by_type": { "choice": 34, "noul": 14, "score": 12 }, "score": 57.31023460546714 }, "usecase": { "accuracy": 0.39662447257383965, "cc": -1.9444617361056398, "cc_by_type": { "choice": 38.716515706301806, "noul": -43.85964912280702, "score": -0.6902517918117024 }, "low_n": false, "n": 237, "n_by_type": { "choice": 111, "noul": 57, "score": 69 }, "score": 0.0 } }, "lang": { "ar": { "accuracy": 0.18181818181818182, "cc": -100.14332542233501, "cc_by_type": { "choice": 20.000000000000004, "noul": -100.0, "score": -220.429976267005 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 4, "score": 2 }, "score": 0.0 }, "cs": { "accuracy": 0.375, "cc": -4.802259887005647, "cc_by_type": { "choice": 23.72881355932204, "noul": -33.333333333333336 }, "low_n": true, "n": 8, "n_by_type": { "choice": 5, "noul": 3 }, "score": 0.0 }, "da": { "accuracy": 0.375, "cc": -8.98770932887382, "cc_by_type": { "choice": 17.159763313609464, "noul": -60.0, "score": 15.877108699769082 }, "low_n": true, "n": 24, "n_by_type": { "choice": 11, "noul": 5, "score": 8 }, "score": 0.0 }, "de": { "accuracy": 0.4222222222222222, "cc": 6.322815228633009, "cc_by_type": { "choice": 34.8219674109837, "noul": -11.111111111111116, "score": -4.742410613973558 }, "low_n": false, "n": 45, "n_by_type": { "choice": 24, "noul": 9, "score": 12 }, "score": 6.322815228633009 }, "el": { "accuracy": 0.2777777777777778, "cc": -39.39660414361751, "cc_by_type": { "choice": 15.662650602409636, "noul": -100.0, "score": -33.85246303326217 }, "low_n": true, "n": 18, "n_by_type": { "choice": 11, "noul": 5, "score": 2 }, "score": 0.0 }, "en": { "accuracy": 0.4924039320822163, "cc": 14.003345625582787, "cc_by_type": { "choice": 44.13909141895677, "noul": -18.045112781954884, "score": 15.916058239746478 }, "low_n": false, "n": 1119, "n_by_type": { "choice": 577, "noul": 266, "score": 276 }, "score": 14.003345625582787 }, "es": { "accuracy": 0.6428571428571429, "cc": 33.758516235110754, "cc_by_type": { "choice": 73.81818181818181, "noul": 27.27272727272727, "score": 0.18463961442317656 }, "low_n": true, "n": 28, "n_by_type": { "choice": 10, "noul": 11, "score": 7 }, "score": 33.758516235110754 }, "fi": { "accuracy": 0.0, "cc": -63.38028169014085, "cc_by_type": { "choice": -26.760563380281692, "noul": -100.0 }, "low_n": true, "n": 9, "n_by_type": { "choice": 6, "noul": 3 }, "score": 0.0 }, "fr": { "accuracy": 0.6, "cc": 28.727877312438604, "cc_by_type": { "choice": 100.00000000000001, "noul": -50.0, "score": 36.183631937315795 }, "low_n": true, "n": 15, "n_by_type": { "choice": 5, "noul": 4, "score": 6 }, "score": 28.727877312438604 }, "hi": { "accuracy": 0.3, "cc": 1.3860870885139314, "cc_by_type": { "choice": 13.043478260869566, "noul": -19.999999999999996, "score": 11.114783004672224 }, "low_n": false, "n": 30, "n_by_type": { "choice": 11, "noul": 10, "score": 9 }, "score": 1.3860870885139314 }, "id": { "accuracy": 0.5, "cc": 17.741935483870968, "cc_by_type": { "choice": 35.483870967741936, "noul": 0.0 }, "low_n": true, "n": 6, "n_by_type": { "choice": 4, "noul": 2 }, "score": 17.741935483870968 }, "it": { "accuracy": 0.6470588235294118, "cc": 40.934769362003244, "cc_by_type": { "choice": 49.22425952045134, "noul": 50.0, "score": 23.5800485655584 }, "low_n": true, "n": 17, "n_by_type": { "choice": 5, "noul": 4, "score": 8 }, "score": 40.934769362003244 }, "ja": { "accuracy": 0.34375, "cc": -7.243137787326186, "cc_by_type": { "choice": 18.69918699186992, "noul": -42.85714285714286, "score": 2.428542503294384 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 0.0 }, "ko": { "accuracy": 0.5454545454545454, "cc": 21.42666905349395, "cc_by_type": { "choice": 34.42622950819673, "noul": 0.0, "score": 29.853777652285117 }, "low_n": true, "n": 11, "n_by_type": { "choice": 4, "noul": 2, "score": 5 }, "score": 21.42666905349395 }, "mixed": { "accuracy": 0.7272727272727273, "cc": 49.321740609526785, "cc_by_type": { "choice": 78.02197802197803, "noul": 0.0, "score": 69.94324380660233 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 4, "score": 1 }, "score": 49.321740609526785 }, "nl": { "accuracy": 0.38461538461538464, "cc": 19.828553848785166, "cc_by_type": { "choice": -0.2785515320334245, "noul": 0.0, "score": 59.76421307838892 }, "low_n": true, "n": 13, "n_by_type": { "choice": 5, "noul": 6, "score": 2 }, "score": 19.828553848785166 }, "no": { "accuracy": 0.42857142857142855, "cc": 17.486338797814206, "cc_by_type": { "choice": 1.6393442622950836, "noul": 33.33333333333333 }, "low_n": true, "n": 7, "n_by_type": { "choice": 4, "noul": 3 }, "score": 17.486338797814206 }, "pl": { "accuracy": 0.525, "cc": 29.992507104770652, "cc_by_type": { "choice": 84.25885439440316, "noul": 0.0, "score": 5.718666919908799 }, "low_n": false, "n": 40, "n_by_type": { "choice": 16, "noul": 10, "score": 14 }, "score": 29.992507104770652 }, "pt": { "accuracy": 0.6666666666666666, "cc": 54.789361628198456, "cc_by_type": { "choice": 63.81909547738693, "noul": 33.33333333333333, "score": 67.2156560738751 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 6, "score": 5 }, "score": 54.789361628198456 }, "sv": { "accuracy": 0.0, "cc": -60.92596627415683, "cc_by_type": { "choice": -33.333333333333336, "noul": -100.0, "score": -49.44456548913716 }, "low_n": true, "n": 8, "n_by_type": { "choice": 3, "noul": 4, "score": 1 }, "score": 0.0 }, "tr": { "accuracy": 0.45454545454545453, "cc": -25.14391404022173, "cc_by_type": { "choice": 73.6842105263158, "noul": -33.333333333333336, "score": -115.78261931364766 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 3, "score": 3 }, "score": 0.0 }, "uk": { "accuracy": 0.5454545454545454, "cc": -3.966886884613037, "cc_by_type": { "choice": 34.78260869565218, "noul": -100.0, "score": 53.3167306505087 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 1, "score": 4 }, "score": 0.0 }, "zh": { "accuracy": 0.5, "cc": 27.7843293385466, "cc_by_type": { "choice": 2.702702702702704, "noul": 33.33333333333333, "score": 47.31695197960376 }, "low_n": true, "n": 8, "n_by_type": { "choice": 4, "noul": 3, "score": 1 }, "score": 27.7843293385466 } }, "type": { "choice": { "accuracy": 0.5546666666666666, "cc": 42.30437035757471, "cc_by_type": { "choice": 42.30437035757471 }, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 42.30437035757471 }, "noul": { "accuracy": 0.408, "cc": -18.400000000000006, "cc_by_type": { "noul": -18.400000000000006 }, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.4026666666666667, "cc": 14.785081876637218, "cc_by_type": { "score": 14.785081876637218 }, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 14.785081876637218 } }, "usecase": { "advertising": { "accuracy": 0.4166666666666667, "cc": -21.322066560908976, "cc_by_type": { "choice": 50.57660626029654, "noul": -60.0, "score": -54.542805943023474 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 0.0 }, "code_linting": { "accuracy": 0.25, "cc": -20.883079422436126, "cc_by_type": { "choice": 1.8003273322422266, "noul": -19.999999999999996, "score": -44.4495655995506 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 0.0 }, "customer_support": { "accuracy": 0.55, "cc": 24.46389210511948, "cc_by_type": { "choice": 83.87096774193549, "noul": -50.0, "score": 39.52070857342296 }, "low_n": true, "n": 20, "n_by_type": { "choice": 8, "noul": 8, "score": 4 }, "score": 24.46389210511948 }, "demand_forecasting": { "accuracy": 0.23076923076923078, "cc": -33.81567016039666, "cc_by_type": { "choice": -8.43373493975904, "noul": -100.0, "score": 6.986724458569061 }, "low_n": true, "n": 26, "n_by_type": { "choice": 11, "noul": 4, "score": 11 }, "score": 0.0 }, "ecommerce_marketplaces": { "accuracy": 0.3888888888888889, "cc": 4.454876806713197, "cc_by_type": { "choice": 44.099378881987576, "noul": -60.0, "score": 29.265251538152015 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 5, "score": 6 }, "score": 4.454876806713197 }, "feature_extraction": { "accuracy": 0.3103448275862069, "cc": -6.573360507662842, "cc_by_type": { "choice": 5.9829059829059785, "noul": -19.999999999999996, "score": -5.702987505894508 }, "low_n": true, "n": 29, "n_by_type": { "choice": 15, "noul": 5, "score": 9 }, "score": 0.0 }, "financial_crime": { "accuracy": 0.46153846153846156, "cc": 16.612884611673458, "cc_by_type": { "choice": 47.82608695652174, "noul": 25.0, "score": -22.987433121501375 }, "low_n": true, "n": 26, "n_by_type": { "choice": 10, "noul": 8, "score": 8 }, "score": 16.612884611673458 }, "gaming": { "accuracy": 0.41379310344827586, "cc": -6.703885975467817, "cc_by_type": { "choice": 40.59405940594059, "noul": -71.42857142857143, "score": 10.722854096227385 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 7, "score": 11 }, "score": 0.0 }, "insurance_claims": { "accuracy": 0.21428571428571427, "cc": -33.44343074859378, "cc_by_type": { "choice": 20.634920634920636, "noul": -100.0, "score": -20.965212880701966 }, "low_n": true, "n": 14, "n_by_type": { "choice": 8, "noul": 5, "score": 1 }, "score": 0.0 }, "knowledge_graphs": { "accuracy": 0.17391304347826086, "cc": -31.072633553703692, "cc_by_type": { "choice": -0.7633587786259586, "noul": -100.0, "score": 7.5454581175148805 }, "low_n": true, "n": 23, "n_by_type": { "choice": 14, "noul": 5, "score": 4 }, "score": 0.0 }, "lead_generation": { "accuracy": 0.32, "cc": -1.078959073185669, "cc_by_type": { "choice": 67.47967479674797, "noul": -75.0, "score": 4.283447983695021 }, "low_n": true, "n": 25, "n_by_type": { "choice": 8, "noul": 8, "score": 9 }, "score": 0.0 }, "legal_compliance": { "accuracy": 0.15384615384615385, "cc": -23.678295476921054, "cc_by_type": { "choice": -14.754098360655735, "noul": -100.0, "score": 43.719211929892566 }, "low_n": true, "n": 13, "n_by_type": { "choice": 8, "noul": 1, "score": 4 }, "score": 0.0 }, "llm_guardrails": { "accuracy": 0.4827586206896552, "cc": 15.868347401942737, "cc_by_type": { "choice": 35.77981651376147, "noul": 25.0, "score": -13.174774307933257 }, "low_n": true, "n": 29, "n_by_type": { "choice": 14, "noul": 8, "score": 7 }, "score": 15.868347401942737 }, "model_routing": { "accuracy": 0.6866666666666666, "cc": 57.504158219489966, "cc_by_type": { "choice": 63.012597158938625, "noul": 59.09090909090908, "score": 50.40896840862219 }, "low_n": false, "n": 150, "n_by_type": { "choice": 74, "noul": 44, "score": 32 }, "score": 57.504158219489966 }, "moderation_trust_safety": { "accuracy": 0.4375, "cc": 7.985104053150214, "cc_by_type": { "choice": 36.84210526315789, "noul": -60.0, "score": 47.113206896292745 }, "low_n": true, "n": 16, "n_by_type": { "choice": 8, "noul": 5, "score": 3 }, "score": 7.985104053150214 }, "recruiting": { "accuracy": 0.38095238095238093, "cc": 0.5006746462399576, "cc_by_type": { "choice": 39.69849246231156, "noul": -66.66666666666667, "score": 28.470198143074978 }, "low_n": true, "n": 21, "n_by_type": { "choice": 9, "noul": 6, "score": 6 }, "score": 0.5006746462399576 }, "risk_assessment": { "accuracy": 0.46153846153846156, "cc": 6.972667050442593, "cc_by_type": { "choice": 43.39622641509434, "noul": -33.333333333333336, "score": 10.855108069566777 }, "low_n": true, "n": 13, "n_by_type": { "choice": 7, "noul": 3, "score": 3 }, "score": 6.972667050442593 }, "scientific_discovery": { "accuracy": 0.25, "cc": -22.282698718421653, "cc_by_type": { "choice": 12.350597609561751, "noul": -71.42857142857143, "score": -7.77012233625527 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 0.0 }, "search_retrieval": { "accuracy": 0.45454545454545453, "cc": -9.604825720085183, "cc_by_type": { "choice": 57.04057279236276, "noul": -100.0, "score": 14.144950047381688 }, "low_n": true, "n": 22, "n_by_type": { "choice": 9, "noul": 6, "score": 7 }, "score": 0.0 } } }, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 70.92554914167138, 79.78637653206756 ], "I": [ 18.019293258402904, 25.188069370363653 ], "capability": [ 45.240972947803556, 51.6024129927801 ], "composite": { "A": [ 5.218755487589073, 12.205081733672536 ], "B": [ 4.193052691245517, 10.348794476091657 ], "C": [ 3.6241357552701894, 8.475751203939263 ] }, "seed": 24 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": false, "g_med_used": 10.073244581339713, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 76.87411794897184, "capability": 49.50665640388703, "intelligence": 22.13919485880221 }, "per_type": { "choice": { "accuracy": 0.5546666666666666, "calibration": 77.37204129541479, "cc": 42.30437035757471, "cc_by_type": { "choice": 42.30437035757471 }, "cc_tiered": 46.38805695454458, "cc_tiered_open": 53.72008003683186, "cc_tiered_sealed": 39.056033872257295, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 42.30437035757471 }, "noul": { "accuracy": 0.408, "calibration": 77.27920881601906, "cc": -18.400000000000006, "cc_by_type": { "noul": -18.400000000000006 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.4026666666666667, "calibration": 75.9711037354817, "cc": 14.785081876637218, "cc_by_type": { "score": 14.785081876637218 }, "cc_tiered": 20.02952762186206, "cc_tiered_open": 29.847390838405925, "cc_tiered_sealed": 10.211664405318189, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 14.785081876637218 } }, "raw_on_A": { "calibration": 72.6387411013128, "capability": 46.47345221333068, "intelligence": 20.308163325348563 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 7.785401938124825, "Emphasis on Accuracy 60:20:20": 5.909418623202946, "Emphasis on Cost 20:20:60": 8.516521034074662, "Emphasis on Speed 20:60:20": 10.132088187487657, "Intelligence only": 4.340556958274228 } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Liquid AI", "axes": { "calibration": 62.32062493293682, "cost": 73.1039158657426, "intelligence": 3.2615852143588597, "speed": 94.0744546594334 }, "calibration": { "parts": { "choice": { "ece_hard": 0.22880094970871764, "ece_score": 54.23981005825647, "mean_tvd": null, "n": 750, "n_ece_hard": 328, "n_tvd": 0, "score": 54.23981005825647 }, "noul": { "abstention_rate": 0.7333333333333334, "brier": 0.2769707015707441, "ece": 0.1533565516303061, "n": 375, "score": 69.32868967393878 }, "score": { "interval90_coverage": 0.8293333333333334, "interval90_width": 0.6586666666666664, "n": 375, "nrps": 0.21825554880108286, "score": 63.39337506661522, "top_ece": 0.25693847493330635 } }, "score": 62.32062493293682 }, "capability": 32.79110507364784, "composite_ci95": { "A": [ 0.00792887171051229, 0.3389815219491588 ], "B": [ 0.005122686535585832, 0.23615583025477924 ], "C": [ 0.005506160910077979, 0.23540383468691578 ] }, "cost": { "basis": "ESTIMATE (*est.), size-class reference; self-hosted open weights, no bookable listing for this base. The declared text base LiquidAI/LFM2.5-Encoder-350M is not a key of any frozen pricing table. Method as above: deepinfra:Qwen/Qwen3.5-0.8B at USD 0.01 per 1M input and USD 0.05 per 1M output - the class above this 600M model - applied to its own measured usage, 1,182,027 input tokens and 0 output tokens over 1,500 decisions. The frozen base-size ENCODER rate (0.005, 0.0), which the published tacet-sonata and lev-350m rows carry, would give 0.00394009 and was deliberately NOT used: it prices a text-only encoder, while d1-omni also runs SigLIP2 vision and FastConformer audio towers, so the class above is the conservative choice and keeps the two Liquid rows on one method.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.00788018 }, "display": "Liquid AI d1-omni-600M", "endpoint_condition": "evaluator-owned Lium RTX A6000 48 GB pod (United States), offline after setup (egress cut and proven), author's custom code at the pinned commit via trust_remote_code from the sha256-verified local copy, float16, states cut on the right at 16,384 tokens by the model code", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 4.293220672965477, "I_sealed": 2.229949755752243, "base": 3.2615852143588597, "excess": -8.009973664126479, "gap": 2.063270917213234, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 12.87966201889643, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 10.486486486486504, "hard": 3.743315508021402, "judge": 12.986156888595916, "standard": 32.18920050230222 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -28.57142857142857, "hard": -73.91304347826086, "judge": -100.0, "standard": -66.66666666666667 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 12.092551202372192, "hard": -10.469003850745073, "judge": 19.089799023912114, "standard": -20.24343611383641 } }, "sealed|choice": { "cc": 6.689849267256729, "n": { "easy": 72, "hard": 260, "judge": 141, "standard": 127 }, "tiers": { "easy": 2.388153092084683, "hard": 3.5396284489392005, "judge": 5.9408452133326675, "standard": 16.2646450723639 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 39, "hard": 132, "judge": 72, "standard": 57 }, "tiers": { "easy": -64.1025641025641, "hard": -72.72727272727273, "judge": -72.22222222222221, "standard": -64.91228070175438 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 33, "hard": 115, "judge": 75, "standard": 77 }, "tiers": { "easy": -14.21064395948739, "hard": -26.382300409685588, "judge": -13.560363505639916, "standard": -9.691900455584058 } } } }, "jevbench_score": 0.049057448753865106, "key": "liquid-d1-omni-600m", "last_measured_on": "2026-10-10", "licence": "LFM Open License v1.0 (commercial use only below USD 10 M annual revenue)", "listing": "ranked", "measured_in": "v1.6.5", "measurement_date_status": "measured", "model_pin": "LiquidAI/d1-omni-600M@02b55d7076f15129e59ab3f94783f32c4b088674", "noul_decisive": { "acc_among_decisive": 0.56, "decisive": 100, "decisive_rate": 0.26666666666666666, "n": 375, "supported": true, "valid": 375 }, "planned_weight_denominator": 1.0, "rank": 14, "ranked": true, "ranks": { "A": 14, "B": 14, "C": 14, "capability": 15 }, "repo": "https://huggingface.co/LiquidAI/d1-omni-600M", "scores": { "A": 0.049057448753865106, "B": 0.032554129042754114, "C": 0.03406767274573965 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.19256855882704257, "p50_s_raw": 0.02128427941352129, "p95_s_adjusted": 0.20322126662358642, "p95_s_raw": 0.026610633311793207 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns": { "family": { "calibration": { "accuracy": 0.20689655172413793, "cc": -29.121942106010263, "cc_by_type": { "choice": -8.333333333333334, "noul": -100.0, "score": 20.967507015302544 }, "low_n": true, "n": 29, "n_by_type": { "choice": 16, "noul": 6, "score": 7 }, "score": 0.0 }, "calibration_gen": { "accuracy": 0.1875, "cc": -48.670081675280734, "cc_by_type": { "choice": -0.1552795031055859, "noul": -76.92307692307692, "score": -68.93188859965969 }, "low_n": false, "n": 112, "n_by_type": { "choice": 56, "noul": 26, "score": 30 }, "score": 0.0 }, "composite_gen": { "accuracy": 0.22448979591836735, "cc": -44.09414082419346, "cc_by_type": { "choice": 11.111111111111109, "noul": -78.57142857142857, "score": -64.82210501226291 }, "low_n": false, "n": 98, "n_by_type": { "choice": 36, "noul": 28, "score": 34 }, "score": 0.0 }, "fairness": { "accuracy": 0.2222222222222222, "cc": -39.3458024428534, "cc_by_type": { "choice": 29.411764705882355, "noul": -100.0, "score": -47.44917203444254 }, "low_n": true, "n": 27, "n_by_type": { "choice": 12, "noul": 6, "score": 9 }, "score": 0.0 }, "jaggedness": { "accuracy": 0.2918918918918919, "cc": -18.904988531894233, "cc_by_type": { "choice": 11.069939786938404, "noul": -64.1025641025641, "score": -3.682341280057 }, "low_n": false, "n": 185, "n_by_type": { "choice": 97, "noul": 39, "score": 49 }, "score": 0.0 }, "llm_router": { "accuracy": 0.3446969696969697, "cc": 1.1432998727522918, "cc_by_type": { "choice": 15.466790524849063, "noul": -28.767123287671236, "score": 16.73023238107905 }, "low_n": false, "n": 264, "n_by_type": { "choice": 131, "noul": 73, "score": 60 }, "score": 1.1432998727522918 }, "long_state": { "accuracy": 0.27586206896551724, "cc": -25.065654575351683, "cc_by_type": { "choice": 16.66666666666667, "noul": -100.0, "score": 8.136369607278283 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 11, "score": 7 }, "score": 0.0 }, "ml_usecase": { "accuracy": 0.22580645161290322, "cc": -33.79734650736005, "cc_by_type": { "choice": 7.692307692307688, "noul": -100.0, "score": -9.084347214387822 }, "low_n": false, "n": 31, "n_by_type": { "choice": 15, "noul": 8, "score": 8 }, "score": 0.0 }, "multilingual": { "accuracy": 0.2, "cc": -31.03237123270632, "cc_by_type": { "choice": -1.2091356918943095, "noul": -93.10344827586206, "score": 1.2154702696374198 }, "low_n": false, "n": 315, "n_by_type": { "choice": 146, "noul": 87, "score": 82 }, "score": 0.0 }, "negation_pair_gen": { "accuracy": 0.16071428571428573, "cc": -85.43494727513347, "cc_by_type": { "choice": -4.761904761904764, "noul": -90.0, "score": -161.54293706349563 }, "low_n": false, "n": 56, "n_by_type": { "choice": 28, "noul": 20, "score": 8 }, "score": 0.0 }, "option_order": { "accuracy": 0.2807017543859649, "cc": 8.99001109877914, "cc_by_type": { "choice": 8.99001109877914 }, "low_n": false, "n": 57, "n_by_type": { "choice": 57 }, "score": 8.99001109877914 }, "perturb_pair": { "accuracy": 0.31666666666666665, "cc": -11.077733022650753, "cc_by_type": { "choice": 21.36304062909567, "noul": -85.71428571428572, "score": 31.1180460172378 }, "low_n": false, "n": 60, "n_by_type": { "choice": 34, "noul": 14, "score": 12 }, "score": 0.0 }, "usecase": { "accuracy": 0.2320675105485232, "cc": -25.394686423610008, "cc_by_type": { "choice": 5.184043168240532, "noul": -57.89473684210527, "score": -23.47336559696529 }, "low_n": false, "n": 237, "n_by_type": { "choice": 111, "noul": 57, "score": 69 }, "score": 0.0 } }, "lang": { "ar": { "accuracy": 0.09090909090909091, "cc": -53.76070940660107, "cc_by_type": { "choice": -33.333333333333336, "noul": -100.0, "score": -27.948794886469862 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 4, "score": 2 }, "score": 0.0 }, "cs": { "accuracy": 0.0, "cc": -63.559322033898304, "cc_by_type": { "choice": -27.11864406779661, "noul": -100.0 }, "low_n": true, "n": 8, "n_by_type": { "choice": 5, "noul": 3 }, "score": 0.0 }, "da": { "accuracy": 0.375, "cc": -22.128997159813196, "cc_by_type": { "choice": 17.159763313609464, "noul": -100.0, "score": 16.45324520695096 }, "low_n": true, "n": 24, "n_by_type": { "choice": 11, "noul": 5, "score": 8 }, "score": 0.0 }, "de": { "accuracy": 0.26666666666666666, "cc": -26.699211006624235, "cc_by_type": { "choice": 7.6644538322269185, "noul": -77.77777777777779, "score": -9.984309074321839 }, "low_n": false, "n": 45, "n_by_type": { "choice": 24, "noul": 9, "score": 12 }, "score": 0.0 }, "el": { "accuracy": 0.16666666666666666, "cc": -44.641658892653076, "cc_by_type": { "choice": -8.43373493975904, "noul": -100.0, "score": -25.491241738200188 }, "low_n": true, "n": 18, "n_by_type": { "choice": 11, "noul": 5, "score": 2 }, "score": 0.0 }, "en": { "accuracy": 0.26273458445040215, "cc": -24.234183293883643, "cc_by_type": { "choice": 8.693213684800824, "noul": -62.40601503759399, "score": -18.989748528857753 }, "low_n": false, "n": 1119, "n_by_type": { "choice": 577, "noul": 266, "score": 276 }, "score": 0.0 }, "es": { "accuracy": 0.21428571428571427, "cc": -23.875465734890014, "cc_by_type": { "choice": 8.363636363636363, "noul": -100.0, "score": 20.00996643169359 }, "low_n": true, "n": 28, "n_by_type": { "choice": 10, "noul": 11, "score": 7 }, "score": 0.0 }, "fi": { "accuracy": 0.2222222222222222, "cc": -42.25352112676057, "cc_by_type": { "choice": 15.49295774647887, "noul": -100.0 }, "low_n": true, "n": 9, "n_by_type": { "choice": 6, "noul": 3 }, "score": 0.0 }, "fr": { "accuracy": 0.2, "cc": -32.05973275794293, "cc_by_type": { "choice": -1.5514809590973198, "noul": -100.0, "score": 5.372282685268514 }, "low_n": true, "n": 15, "n_by_type": { "choice": 5, "noul": 4, "score": 6 }, "score": 0.0 }, "hi": { "accuracy": 0.2, "cc": -27.81219448998868, "cc_by_type": { "choice": 13.043478260869566, "noul": -100.0, "score": 3.519938269164402 }, "low_n": false, "n": 30, "n_by_type": { "choice": 11, "noul": 10, "score": 9 }, "score": 0.0 }, "id": { "accuracy": 0.16666666666666666, "cc": -48.387096774193544, "cc_by_type": { "choice": 3.225806451612906, "noul": -100.0 }, "low_n": true, "n": 6, "n_by_type": { "choice": 4, "noul": 2 }, "score": 0.0 }, "it": { "accuracy": 0.23529411764705882, "cc": -47.53372129710758, "cc_by_type": { "choice": 23.83638928067701, "noul": -100.0, "score": -66.43755317199978 }, "low_n": true, "n": 17, "n_by_type": { "choice": 5, "noul": 4, "score": 8 }, "score": 0.0 }, "ja": { "accuracy": 0.125, "cc": -47.14183058522813, "cc_by_type": { "choice": -13.821138211382111, "noul": -100.0, "score": -27.604353544302263 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 0.0 }, "ko": { "accuracy": 0.18181818181818182, "cc": -43.344897587116954, "cc_by_type": { "choice": -31.14754098360656, "noul": -100.0, "score": 1.1128482222557023 }, "low_n": true, "n": 11, "n_by_type": { "choice": 4, "noul": 2, "score": 5 }, "score": 0.0 }, "mixed": { "accuracy": 0.45454545454545453, "cc": 16.030320645251987, "cc_by_type": { "choice": 56.04395604395604, "noul": -50.0, "score": 42.04700589179993 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 4, "score": 1 }, "score": 16.030320645251987 }, "nl": { "accuracy": 0.23076923076923078, "cc": -20.35052023493665, "cc_by_type": { "choice": 24.79108635097493, "noul": -66.66666666666667, "score": -19.17598038911821 }, "low_n": true, "n": 13, "n_by_type": { "choice": 5, "noul": 6, "score": 2 }, "score": 0.0 }, "no": { "accuracy": 0.14285714285714285, "cc": -49.18032786885246, "cc_by_type": { "choice": 1.6393442622950836, "noul": -100.0 }, "low_n": true, "n": 7, "n_by_type": { "choice": 4, "noul": 3 }, "score": 0.0 }, "pl": { "accuracy": 0.225, "cc": -12.618486802534166, "cc_by_type": { "choice": 5.553126366418887, "noul": -60.0, "score": 16.591413225978613 }, "low_n": false, "n": 40, "n_by_type": { "choice": 16, "noul": 10, "score": 14 }, "score": 0.0 }, "pt": { "accuracy": 0.2222222222222222, "cc": -28.382395955071996, "cc_by_type": { "choice": 9.547738693467334, "noul": -66.66666666666667, "score": -28.02825989201665 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 6, "score": 5 }, "score": 0.0 }, "sv": { "accuracy": 0.0, "cc": -48.522135284211906, "cc_by_type": { "choice": -33.333333333333336, "noul": -100.0, "score": -12.233072519302368 }, "low_n": true, "n": 8, "n_by_type": { "choice": 3, "noul": 4, "score": 1 }, "score": 0.0 }, "tr": { "accuracy": 0.36363636363636365, "cc": -18.188721011256614, "cc_by_type": { "choice": -5.26315789473684, "noul": -100.0, "score": 50.696994860967 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 3, "score": 3 }, "score": 0.0 }, "uk": { "accuracy": 0.09090909090909091, "cc": -41.07722171240693, "cc_by_type": { "choice": -30.43478260869565, "noul": -100.0, "score": 7.2031174714748625 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 1, "score": 4 }, "score": 0.0 }, "zh": { "accuracy": 0.375, "cc": -10.272994610640382, "cc_by_type": { "choice": 35.13513513513514, "noul": -100.0, "score": 34.045881032943726 }, "low_n": true, "n": 8, "n_by_type": { "choice": 4, "noul": 3, "score": 1 }, "score": 0.0 } }, "type": { "choice": { "accuracy": 0.2853333333333333, "cc": 7.410606322335463, "cc_by_type": { "choice": 7.410606322335463 }, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 7.410606322335463 }, "noul": { "accuracy": 0.14933333333333335, "cc": -70.13333333333334, "cc_by_type": { "noul": -70.13333333333334 }, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.2853333333333333, "cc": -14.348760038030782, "cc_by_type": { "score": -14.348760038030782 }, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "usecase": { "advertising": { "accuracy": 0.2916666666666667, "cc": -18.14065897262343, "cc_by_type": { "choice": -8.731466227347617, "noul": -19.999999999999996, "score": -25.690510690522682 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 0.0 }, "code_linting": { "accuracy": 0.16666666666666666, "cc": -19.845708892643234, "cc_by_type": { "choice": -8.01963993453355, "noul": -60.0, "score": 8.482513256603841 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 0.0 }, "customer_support": { "accuracy": 0.2, "cc": -36.093708917165806, "cc_by_type": { "choice": 3.225806451612906, "noul": -75.0, "score": -36.506933203110336 }, "low_n": true, "n": 20, "n_by_type": { "choice": 8, "noul": 8, "score": 4 }, "score": 0.0 }, "demand_forecasting": { "accuracy": 0.19230769230769232, "cc": -43.411243099056556, "cc_by_type": { "choice": 15.662650602409636, "noul": -100.0, "score": -45.89637989957931 }, "low_n": true, "n": 26, "n_by_type": { "choice": 11, "noul": 4, "score": 11 }, "score": 0.0 }, "ecommerce_marketplaces": { "accuracy": 0.2222222222222222, "cc": -19.47483978171899, "cc_by_type": { "choice": 6.832298136645961, "noul": -60.0, "score": -5.256817481802933 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 5, "score": 6 }, "score": 0.0 }, "feature_extraction": { "accuracy": 0.10344827586206896, "cc": -52.342876150533066, "cc_by_type": { "choice": -11.111111111111114, "noul": -100.0, "score": -45.917517340488075 }, "low_n": true, "n": 29, "n_by_type": { "choice": 15, "noul": 5, "score": 9 }, "score": 0.0 }, "financial_crime": { "accuracy": 0.38461538461538464, "cc": 0.6176093078177957, "cc_by_type": { "choice": 8.695652173913043, "noul": 0.0, "score": -6.842824250459656 }, "low_n": true, "n": 26, "n_by_type": { "choice": 10, "noul": 8, "score": 8 }, "score": 0.6176093078177957 }, "gaming": { "accuracy": 0.20689655172413793, "cc": -26.6661863971574, "cc_by_type": { "choice": 16.831683168316832, "noul": -71.42857142857143, "score": -25.40167093121759 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 7, "score": 11 }, "score": 0.0 }, "insurance_claims": { "accuracy": 0.2857142857142857, "cc": 4.412051451900017, "cc_by_type": { "choice": 20.634920634920636, "noul": -100.0, "score": 92.60123372077942 }, "low_n": true, "n": 14, "n_by_type": { "choice": 8, "noul": 5, "score": 1 }, "score": 4.412051451900017 }, "knowledge_graphs": { "accuracy": 0.08695652173913043, "cc": -55.71206240218595, "cc_by_type": { "choice": -9.923664122137408, "noul": -100.0, "score": -57.21252308442044 }, "low_n": true, "n": 23, "n_by_type": { "choice": 14, "noul": 5, "score": 4 }, "score": 0.0 }, "lead_generation": { "accuracy": 0.28, "cc": -16.228836402216572, "cc_by_type": { "choice": 18.69918699186992, "noul": -100.0, "score": 32.61430380148036 }, "low_n": true, "n": 25, "n_by_type": { "choice": 8, "noul": 8, "score": 9 }, "score": 0.0 }, "legal_compliance": { "accuracy": 0.23076923076923078, "cc": -29.703887276787327, "cc_by_type": { "choice": 1.6393442622950836, "noul": -100.0, "score": 9.248993907342928 }, "low_n": true, "n": 13, "n_by_type": { "choice": 8, "noul": 1, "score": 4 }, "score": 0.0 }, "llm_guardrails": { "accuracy": 0.3103448275862069, "cc": -15.934116861404396, "cc_by_type": { "choice": 17.431192660550458, "noul": -50.0, "score": -15.233543244763649 }, "low_n": true, "n": 29, "n_by_type": { "choice": 14, "noul": 8, "score": 7 }, "score": 0.0 }, "model_routing": { "accuracy": 0.4, "cc": 4.006137375452837, "cc_by_type": { "choice": 27.633342267488622, "noul": -40.90909090909091, "score": 25.294160767960793 }, "low_n": false, "n": 150, "n_by_type": { "choice": 74, "noul": 44, "score": 32 }, "score": 4.006137375452837 }, "moderation_trust_safety": { "accuracy": 0.25, "cc": -29.448869234338442, "cc_by_type": { "choice": 5.263157894736842, "noul": -100.0, "score": 6.390234402247841 }, "low_n": true, "n": 16, "n_by_type": { "choice": 8, "noul": 5, "score": 3 }, "score": 0.0 }, "recruiting": { "accuracy": 0.23809523809523808, "cc": -26.169667953406684, "cc_by_type": { "choice": 24.62311557788945, "noul": -100.0, "score": -3.132119438109493 }, "low_n": true, "n": 21, "n_by_type": { "choice": 9, "noul": 6, "score": 6 }, "score": 0.0 }, "risk_assessment": { "accuracy": 0.23076923076923078, "cc": -41.00485732486038, "cc_by_type": { "choice": 5.660377358490565, "noul": -100.0, "score": -28.67494933307171 }, "low_n": true, "n": 13, "n_by_type": { "choice": 7, "noul": 3, "score": 3 }, "score": 0.0 }, "scientific_discovery": { "accuracy": 0.15625, "cc": -22.981094893304945, "cc_by_type": { "choice": -11.553784860557771, "noul": -71.42857142857143, "score": 14.03907160921436 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 0.0 }, "search_retrieval": { "accuracy": 0.13636363636363635, "cc": -52.2527400518963, "cc_by_type": { "choice": -14.558472553699291, "noul": -100.0, "score": -42.199747601989635 }, "low_n": true, "n": 22, "n_by_type": { "choice": 9, "noul": 6, "score": 7 }, "score": 0.0 } } }, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 56.7149821052611, 66.00061446604302 ], "I": [ 1.7436767620361024, 6.437188407255578 ], "capability": [ 29.712243529714982, 35.427303807623055 ], "composite": { "A": [ 0.00792887171051229, 0.3389815219491588 ], "B": [ 0.005122686535585832, 0.23615583025477924 ], "C": [ 0.005506160910077979, 0.23540383468691578 ] }, "seed": 25 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": false, "g_med_used": 10.073244581339713, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 62.32062493293682, "capability": 32.79110507364784, "intelligence": 3.2615852143588597 }, "per_type": { "choice": { "accuracy": 0.2853333333333333, "calibration": 54.23981005825647, "cc": 7.410606322335463, "cc_by_type": { "choice": 7.410606322335463 }, "cc_tiered": 9.78475564307658, "cc_tiered_open": 12.87966201889643, "cc_tiered_sealed": 6.689849267256729, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 7.410606322335463 }, "noul": { "accuracy": 0.14933333333333335, "calibration": 69.32868967393878, "cc": -70.13333333333334, "cc_by_type": { "noul": -70.13333333333334 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.2853333333333333, "calibration": 63.39337506661522, "cc": -14.348760038030782, "cc_by_type": { "score": -14.348760038030782 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 60.8515976034804, "capability": 31.49910396998157, "intelligence": 2.1466103364827385 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.038577217040809395, "Emphasis on Accuracy 60:20:20": 0.022535446033063476, "Emphasis on Cost 20:20:60": 0.05938557623710964, "Emphasis on Speed 20:60:20": 0.06041396685797389, "Intelligence only": 0.013878616661020313 } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "ksonny4 / @kubelpe (ksonny4@gmail.com)", "axes": { "calibration": 66.68251917899094, "cost": 70.36992715953406, "intelligence": 2.6107469083924393, "speed": 57.79924670989358 }, "calibration": { "parts": { "choice": { "ece_hard": 0.16197772093040966, "ece_score": 67.60445581391808, "mean_tvd": null, "n": 750, "n_ece_hard": 328, "n_tvd": 0, "score": 67.60445581391808 }, "noul": { "abstention_rate": 1.0, "brier": 0.28875559863871086, "ece": 0.20363870372723708, "n": 375, "score": 59.27225925455259 }, "score": { "interval90_coverage": 0.96, "interval90_width": 0.8991111111111114, "n": 375, "nrps": 0.1846962110408624, "score": 73.17084246850217, "top_ece": 0.17594346979454697 } }, "score": 66.68251917899094 }, "capability": 34.64663304369169, "composite_ci95": { "A": [ 0.0008967609092971513, 0.12984410368517996 ], "B": [ 0.0005711694021380616, 0.08896822577663006 ], "C": [ 0.000622750631456355, 0.09016951644804166 ] }, "cost": { "basis": "ESTIMATE (*est.), size-class reference; self-hosted open weights, no public tariff. The same choice the published quyet-1-0-small-en row carries for the identical ModernBERT-base backbone: DeepInfra base-size encoders at USD 0.005 per 1M input and 0 output (deepinfra:thenlper/gte-base, BAAI/bge-base-en-v1.5, intfloat/e5-base-v2 and sentence-transformers/all-mpnet-base-v2 are all 5e-7 cents per input token in the frozen 25 Sep 2026 DeepInfra snapshot). Output tokens are a literal 0 reported by the author's server (NLI entailment read-out, nothing is generated). This row's own usage on draw v1.6-regular-20261010-a2: 2,916,019 input and 0 output tokens over 1,500 decisions. Estimated, not charged.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.009720063333333334 }, "display": "modernbert-jev v2", "endpoint_condition": "Sandy, the evaluator's own shared CPU host (AMD Ryzen 5 3600, 6 cores / 12 threads, Zen 2, AVX2), NOT a dedicated machine: other jobs ran beside it and the 1-minute load average was logged every 60 s. The author's own CLI at github.com/KSonny4/modernbert-jev@b1866e4d, `modernbert-jev serve --model ` at its own defaults (1 ONNX Runtime thread, 127.0.0.1), inside a loopback-only network namespace with an outbound probe that failed as required, no credentials in the environment. int8 (unsigned) ONNX as shipped. Speed is a lower bound for this model on a dedicated core.", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 3.8269990036680306, "I_sealed": 1.394494813116848, "base": 2.6107469083924393, "excess": -7.6407403907885305, "gap": 2.4325041905511826, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 11.480997011004092, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 10.486486486486504, "hard": -1.9188424032714584, "judge": 20.89650626235992, "standard": 24.65466722478024 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 4.480029631602289, "hard": -0.07960342036494872, "judge": 1.7876323239060077, "standard": -15.095565734194793 } }, "sealed|choice": { "cc": 4.183484439350544, "n": { "easy": 72, "hard": 260, "judge": 141, "standard": 127 }, "tiers": { "easy": 8.019605798310561, "hard": 2.055622732769036, "judge": 3.2012581807112888, "standard": 7.994486560992435 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 39, "hard": 132, "judge": 72, "standard": 57 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 33, "hard": 115, "judge": 75, "standard": 77 }, "tiers": { "easy": 1.478176928885322, "hard": -3.5620793106463378, "judge": -5.974082989668261, "standard": -11.414183075878537 } } } }, "jevbench_score": 0.025388991293562355, "key": "modernbert-jev-v2", "last_measured_on": "2026-10-10", "licence": "Apache-2.0 (model card; the code README and pyproject.toml also declare Apache-2.0, without a separate LICENSE file)", "listing": "ranked", "measured_in": "v1.6.8", "measurement_date_status": "measured", "model_pin": "ksonny4/modernbert-jev-v2@9708136f22a85b29fcf90d3960aa642d9eadcb1c", "noul_decisive": { "acc_among_decisive": null, "decisive": 0, "decisive_rate": 0.0, "n": 375, "supported": true, "valid": 375 }, "planned_weight_denominator": 1.0, "rank": 16, "ranked": true, "ranks": { "A": 16, "B": 16, "C": 16, "capability": 13 }, "repo": "https://huggingface.co/ksonny4/modernbert-jev-v2", "scores": { "A": 0.025388991293562355, "B": 0.016776345440377434, "C": 0.01763124395386275 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 5.349294321238995, "p50_s_raw": 2.5996471606194973, "p95_s_adjusted": 31.02978992164135, "p95_s_raw": 15.439894960820675 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns": { "family": { "calibration": { "accuracy": 0.13793103448275862, "cc": -43.39863396756374, "cc_by_type": { "choice": -8.333333333333334, "noul": -100.0, "score": -21.862568569357887 }, "low_n": true, "n": 29, "n_by_type": { "choice": 16, "noul": 6, "score": 7 }, "score": 0.0 }, "calibration_gen": { "accuracy": 0.17857142857142858, "cc": -45.8159894104131, "cc_by_type": { "choice": 2.173913043478264, "noul": -100.0, "score": -39.621881274717595 }, "low_n": false, "n": 112, "n_by_type": { "choice": 56, "noul": 26, "score": 30 }, "score": 0.0 }, "composite_gen": { "accuracy": 0.22448979591836735, "cc": -32.536005477410086, "cc_by_type": { "choice": 3.7037037037037055, "noul": -100.0, "score": -1.3117201359339692 }, "low_n": false, "n": 98, "n_by_type": { "choice": 36, "noul": 28, "score": 34 }, "score": 0.0 }, "fairness": { "accuracy": 0.3333333333333333, "cc": -25.18773661803903, "cc_by_type": { "choice": -5.882352941176475, "noul": -100.0, "score": 30.319143087059388 }, "low_n": true, "n": 27, "n_by_type": { "choice": 12, "noul": 6, "score": 9 }, "score": 0.0 }, "jaggedness": { "accuracy": 0.14594594594594595, "cc": -46.20038451819038, "cc_by_type": { "choice": -12.552107457156087, "noul": -100.0, "score": -26.04904609741505 }, "low_n": false, "n": 185, "n_by_type": { "choice": 97, "noul": 39, "score": 49 }, "score": 0.0 }, "llm_router": { "accuracy": 0.18181818181818182, "cc": -31.90735375118363, "cc_by_type": { "choice": 10.822108685555056, "noul": -100.0, "score": -6.544169939105937 }, "low_n": false, "n": 264, "n_by_type": { "choice": 131, "noul": 73, "score": 60 }, "score": 0.0 }, "long_state": { "accuracy": 0.034482758620689655, "cc": -44.327454537857705, "cc_by_type": { "choice": -30.952380952380945, "noul": -100.0, "score": -2.0299826611921645 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 11, "score": 7 }, "score": 0.0 }, "ml_usecase": { "accuracy": 0.1935483870967742, "cc": -36.52576519511609, "cc_by_type": { "choice": -1.5384615384615408, "noul": -100.0, "score": -8.038834046886723 }, "low_n": false, "n": 31, "n_by_type": { "choice": 15, "noul": 8, "score": 8 }, "score": 0.0 }, "multilingual": { "accuracy": 0.20952380952380953, "cc": -28.34675764556054, "cc_by_type": { "choice": 6.8517689207344405, "noul": -100.0, "score": 8.107958142583938 }, "low_n": false, "n": 315, "n_by_type": { "choice": 146, "noul": 87, "score": 82 }, "score": 0.0 }, "negation_pair_gen": { "accuracy": 0.14285714285714285, "cc": -56.58208630213584, "cc_by_type": { "choice": -4.761904761904764, "noul": -100.0, "score": -64.98435414450276 }, "low_n": false, "n": 56, "n_by_type": { "choice": 28, "noul": 20, "score": 8 }, "score": 0.0 }, "option_order": { "accuracy": 0.21052631578947367, "cc": 0.11098779134295962, "cc_by_type": { "choice": 0.11098779134295962 }, "low_n": false, "n": 57, "n_by_type": { "choice": 57 }, "score": 0.11098779134295962 }, "perturb_pair": { "accuracy": 0.2833333333333333, "cc": -18.717250201930284, "cc_by_type": { "choice": 17.431192660550455, "noul": -100.0, "score": 26.417056733658693 }, "low_n": false, "n": 60, "n_by_type": { "choice": 34, "noul": 14, "score": 12 }, "score": 0.0 }, "usecase": { "accuracy": 0.2489451476793249, "cc": -25.635511113946517, "cc_by_type": { "choice": 21.37213335902873, "noul": -100.0, "score": 1.7213332991317332 }, "low_n": false, "n": 237, "n_by_type": { "choice": 111, "noul": 57, "score": 69 }, "score": 0.0 } }, "lang": { "ar": { "accuracy": 0.2727272727272727, "cc": -34.51061721771425, "cc_by_type": { "choice": -6.666666666666665, "noul": -100.0, "score": 3.134815013523906 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 4, "score": 2 }, "score": 0.0 }, "cs": { "accuracy": 0.125, "cc": -50.847457627118644, "cc_by_type": { "choice": -1.694915254237285, "noul": -100.0 }, "low_n": true, "n": 8, "n_by_type": { "choice": 5, "noul": 3 }, "score": 0.0 }, "da": { "accuracy": 0.3333333333333333, "cc": -20.541065093618435, "cc_by_type": { "choice": 40.8284023668639, "noul": -100.0, "score": -2.451597647719206 }, "low_n": true, "n": 24, "n_by_type": { "choice": 11, "noul": 5, "score": 8 }, "score": 0.0 }, "de": { "accuracy": 0.2, "cc": -32.195675052815936, "cc_by_type": { "choice": 2.232951116475559, "noul": -100.0, "score": 1.1800237250766288 }, "low_n": false, "n": 45, "n_by_type": { "choice": 24, "noul": 9, "score": 12 }, "score": 0.0 }, "el": { "accuracy": 0.16666666666666666, "cc": -50.44316644371644, "cc_by_type": { "choice": 3.614457831325295, "noul": -100.0, "score": -54.94395716247462 }, "low_n": true, "n": 18, "n_by_type": { "choice": 11, "noul": 5, "score": 2 }, "score": 0.0 }, "en": { "accuracy": 0.19571045576407506, "cc": -34.73453804072174, "cc_by_type": { "choice": 4.206393718451972, "noul": -100.0, "score": -8.410007840617183 }, "low_n": false, "n": 1119, "n_by_type": { "choice": 577, "noul": 266, "score": 276 }, "score": 0.0 }, "es": { "accuracy": 0.25, "cc": -18.257637042739912, "cc_by_type": { "choice": 21.45454545454546, "noul": -100.0, "score": 23.772543417234793 }, "low_n": true, "n": 28, "n_by_type": { "choice": 10, "noul": 11, "score": 7 }, "score": 0.0 }, "fi": { "accuracy": 0.2222222222222222, "cc": -42.25352112676057, "cc_by_type": { "choice": 15.49295774647887, "noul": -100.0 }, "low_n": true, "n": 9, "n_by_type": { "choice": 6, "noul": 3 }, "score": 0.0 }, "fr": { "accuracy": 0.4, "cc": -12.310038757433736, "cc_by_type": { "choice": 74.61212976022568, "noul": -100.0, "score": -11.542246032526894 }, "low_n": true, "n": 15, "n_by_type": { "choice": 5, "noul": 4, "score": 6 }, "score": 0.0 }, "hi": { "accuracy": 0.13333333333333333, "cc": -35.49650623401251, "cc_by_type": { "choice": -11.801242236024846, "noul": -100.0, "score": 5.3117235339873226 }, "low_n": false, "n": 30, "n_by_type": { "choice": 11, "noul": 10, "score": 9 }, "score": 0.0 }, "id": { "accuracy": 0.3333333333333333, "cc": -32.25806451612903, "cc_by_type": { "choice": 35.483870967741936, "noul": -100.0 }, "low_n": true, "n": 6, "n_by_type": { "choice": 4, "noul": 2 }, "score": 0.0 }, "it": { "accuracy": 0.29411764705882354, "cc": -22.98844453863286, "cc_by_type": { "choice": 23.83638928067701, "noul": -100.0, "score": 7.198277103424422 }, "low_n": true, "n": 17, "n_by_type": { "choice": 5, "noul": 4, "score": 8 }, "score": 0.0 }, "ja": { "accuracy": 0.125, "cc": -40.139871804621805, "cc_by_type": { "choice": -13.821138211382111, "noul": -100.0, "score": -6.598477202483299 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 0.0 }, "ko": { "accuracy": 0.2727272727272727, "cc": -36.975867376244004, "cc_by_type": { "choice": 1.6393442622950836, "noul": -100.0, "score": -12.56694639102709 }, "low_n": true, "n": 11, "n_by_type": { "choice": 4, "noul": 2, "score": 5 }, "score": 0.0 }, "mixed": { "accuracy": 0.0, "cc": -40.20521661272176, "cc_by_type": { "choice": -31.868131868131872, "noul": -100.0, "score": 11.252482029966604 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 4, "score": 1 }, "score": 0.0 }, "nl": { "accuracy": 0.07692307692307693, "cc": -29.998198543905005, "cc_by_type": { "choice": -0.2785515320334245, "noul": -100.0, "score": 10.283955900318398 }, "low_n": true, "n": 13, "n_by_type": { "choice": 5, "noul": 6, "score": 2 }, "score": 0.0 }, "no": { "accuracy": 0.2857142857142857, "cc": -32.78688524590164, "cc_by_type": { "choice": 34.42622950819673, "noul": -100.0 }, "low_n": true, "n": 7, "n_by_type": { "choice": 4, "noul": 3 }, "score": 0.0 }, "pl": { "accuracy": 0.2, "cc": -24.81588086706398, "cc_by_type": { "choice": 5.553126366418887, "noul": -100.0, "score": 19.999231032389165 }, "low_n": false, "n": 40, "n_by_type": { "choice": 16, "noul": 10, "score": 14 }, "score": 0.0 }, "pt": { "accuracy": 0.2777777777777778, "cc": -9.14274565490507, "cc_by_type": { "choice": 63.81909547738693, "noul": -100.0, "score": 8.752667557897864 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 6, "score": 5 }, "score": 0.0 }, "sv": { "accuracy": 0.0, "cc": -43.590934068488735, "cc_by_type": { "choice": -33.333333333333336, "noul": -100.0, "score": 2.5605311278671383 }, "low_n": true, "n": 8, "n_by_type": { "choice": 3, "noul": 4, "score": 1 }, "score": 0.0 }, "tr": { "accuracy": 0.36363636363636365, "cc": -17.28662918547593, "cc_by_type": { "choice": 21.052631578947373, "noul": -100.0, "score": 27.087480864624837 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 3, "score": 3 }, "score": 0.0 }, "uk": { "accuracy": 0.2727272727272727, "cc": -23.219083224832847, "cc_by_type": { "choice": -8.695652173913041, "noul": -100.0, "score": 39.038402499414495 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 1, "score": 4 }, "score": 0.0 }, "zh": { "accuracy": 0.0, "cc": -44.80276763421367, "cc_by_type": { "choice": -29.729729729729726, "noul": -100.0, "score": -4.678573172911271 }, "low_n": true, "n": 8, "n_by_type": { "choice": 4, "noul": 3, "score": 1 }, "score": 0.0 } }, "type": { "choice": { "accuracy": 0.268, "cc": 5.164968042839871, "cc_by_type": { "choice": 5.164968042839871 }, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 5.164968042839871 }, "noul": { "accuracy": 0.0, "cc": -100.0, "cc_by_type": { "noul": -100.0 }, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.2613333333333333, "cc": -4.528144639423148, "cc_by_type": { "score": -4.528144639423148 }, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "usecase": { "advertising": { "accuracy": 0.375, "cc": -20.45434194583145, "cc_by_type": { "choice": 30.807248764415156, "noul": -100.0, "score": 7.829725398090492 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 0.0 }, "code_linting": { "accuracy": 0.2916666666666667, "cc": -23.589501736030243, "cc_by_type": { "choice": 31.26022913256956, "noul": -100.0, "score": -2.0287343406602876 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 0.0 }, "customer_support": { "accuracy": 0.15, "cc": -30.19926598996379, "cc_by_type": { "choice": 3.225806451612906, "noul": -100.0, "score": 6.176395578495719 }, "low_n": true, "n": 20, "n_by_type": { "choice": 8, "noul": 8, "score": 4 }, "score": 0.0 }, "demand_forecasting": { "accuracy": 0.15384615384615385, "cc": -40.108407064559685, "cc_by_type": { "choice": -8.43373493975904, "noul": -100.0, "score": -11.891486253920025 }, "low_n": true, "n": 26, "n_by_type": { "choice": 11, "noul": 4, "score": 11 }, "score": 0.0 }, "ecommerce_marketplaces": { "accuracy": 0.2777777777777778, "cc": -17.392819381831114, "cc_by_type": { "choice": 44.099378881987576, "noul": -100.0, "score": 3.722162972519083 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 5, "score": 6 }, "score": 0.0 }, "feature_extraction": { "accuracy": 0.1724137931034483, "cc": -35.42851137104405, "cc_by_type": { "choice": -2.564102564102566, "noul": -100.0, "score": -3.721431549029597 }, "low_n": true, "n": 29, "n_by_type": { "choice": 15, "noul": 5, "score": 9 }, "score": 0.0 }, "financial_crime": { "accuracy": 0.19230769230769232, "cc": -26.23012292511098, "cc_by_type": { "choice": 8.695652173913043, "noul": -100.0, "score": 12.61397905075401 }, "low_n": true, "n": 26, "n_by_type": { "choice": 10, "noul": 8, "score": 8 }, "score": 0.0 }, "gaming": { "accuracy": 0.1724137931034483, "cc": -30.62771796722504, "cc_by_type": { "choice": 4.950495049504946, "noul": -100.0, "score": 3.1663510488199376 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 7, "score": 11 }, "score": 0.0 }, "insurance_claims": { "accuracy": 0.42857142857142855, "cc": 9.59819138604467, "cc_by_type": { "choice": 52.38095238095238, "noul": -100.0, "score": 76.41362177718163 }, "low_n": true, "n": 14, "n_by_type": { "choice": 8, "noul": 5, "score": 1 }, "score": 9.59819138604467 }, "knowledge_graphs": { "accuracy": 0.13043478260869565, "cc": -38.28700352328199, "cc_by_type": { "choice": -0.7633587786259586, "noul": -100.0, "score": -14.09765179122 }, "low_n": true, "n": 23, "n_by_type": { "choice": 14, "noul": 5, "score": 4 }, "score": 0.0 }, "lead_generation": { "accuracy": 0.16, "cc": -32.661969492218304, "cc_by_type": { "choice": 2.4390243902439046, "noul": -100.0, "score": -0.4249328668988195 }, "low_n": true, "n": 25, "n_by_type": { "choice": 8, "noul": 8, "score": 9 }, "score": 0.0 }, "legal_compliance": { "accuracy": 0.38461538461538464, "cc": -15.740475832931393, "cc_by_type": { "choice": 34.42622950819673, "noul": -100.0, "score": 18.352342993009096 }, "low_n": true, "n": 13, "n_by_type": { "choice": 8, "noul": 1, "score": 4 }, "score": 0.0 }, "llm_guardrails": { "accuracy": 0.3448275862068966, "cc": -22.60830157830547, "cc_by_type": { "choice": 26.60550458715596, "noul": -100.0, "score": 5.569590677927638 }, "low_n": true, "n": 29, "n_by_type": { "choice": 14, "noul": 8, "score": 7 }, "score": 0.0 }, "model_routing": { "accuracy": 0.21333333333333335, "cc": -26.190070048541028, "cc_by_type": { "choice": 19.592602519431804, "noul": -100.0, "score": 1.8371873349451118 }, "low_n": false, "n": 150, "n_by_type": { "choice": 74, "noul": 44, "score": 32 }, "score": 0.0 }, "moderation_trust_safety": { "accuracy": 0.5, "cc": -3.248277111101165, "cc_by_type": { "choice": 68.42105263157895, "noul": -100.0, "score": 21.834116035117557 }, "low_n": true, "n": 16, "n_by_type": { "choice": 8, "noul": 5, "score": 3 }, "score": 0.0 }, "recruiting": { "accuracy": 0.19047619047619047, "cc": -28.752891240849554, "cc_by_type": { "choice": 9.547738693467341, "noul": -100.0, "score": 4.193587583984004 }, "low_n": true, "n": 21, "n_by_type": { "choice": 9, "noul": 6, "score": 6 }, "score": 0.0 }, "risk_assessment": { "accuracy": 0.15384615384615385, "cc": -37.98670463078509, "cc_by_type": { "choice": 5.660377358490565, "noul": -100.0, "score": -19.620491250845838 }, "low_n": true, "n": 13, "n_by_type": { "choice": 7, "noul": 3, "score": 3 }, "score": 0.0 }, "scientific_discovery": { "accuracy": 0.15625, "cc": -32.51983448550218, "cc_by_type": { "choice": -3.585657370517929, "noul": -100.0, "score": 6.0261539140113785 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 0.0 }, "search_retrieval": { "accuracy": 0.09090909090909091, "cc": -35.388151349291284, "cc_by_type": { "choice": -14.558472553699291, "noul": -100.0, "score": 8.394018505825429 }, "low_n": true, "n": 22, "n_by_type": { "choice": 9, "noul": 6, "score": 7 }, "score": 0.0 } } }, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 61.644678187031296, 70.74968953890287 ], "I": [ 0.8350318363608307, 4.617476925221922 ], "capability": [ 31.94442581392808, 37.14219934722527 ], "composite": { "A": [ 0.0008967609092971513, 0.12984410368517996 ], "B": [ 0.0005711694021380616, 0.08896822577663006 ], "C": [ 0.000622750631456355, 0.09016951644804166 ] }, "seed": 26 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": false, "g_med_used": 10.073244581339713, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 66.68251917899094, "capability": 34.64663304369169, "intelligence": 2.6107469083924393 }, "per_type": { "choice": { "accuracy": 0.268, "calibration": 67.60445581391808, "cc": 5.164968042839871, "cc_by_type": { "choice": 5.164968042839871 }, "cc_tiered": 7.832240725177318, "cc_tiered_open": 11.480997011004092, "cc_tiered_sealed": 4.183484439350544, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 5.164968042839871 }, "noul": { "accuracy": 0.0, "calibration": 59.27225925455259, "cc": -100.0, "cc_by_type": { "noul": -100.0 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.2613333333333333, "calibration": 73.17084246850217, "cc": -4.528144639423148, "cc_by_type": { "score": -4.528144639423148 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 59.34147162065773, "capability": 31.140405932959595, "intelligence": 2.939340245261466 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.01973059249456439, "Emphasis on Accuracy 60:20:20": 0.011546588753666046, "Emphasis on Cost 20:20:60": 0.030774419703851856, "Emphasis on Speed 20:60:20": 0.030350892509696595, "Intelligence only": 0.0071179397650143975 } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Shanghua Gao (@GaoShanghua)", "axes": { "calibration": 85.34979344070516, "cost": 31.141652902933714, "intelligence": 61.564156247760415, "speed": 84.71397705257328 }, "calibration": { "parts": { "choice": { "ece_hard": 0.08475596558784464, "ece_score": 83.04880688243107, "mean_tvd": null, "n": 739, "n_ece_hard": 317, "n_tvd": 0, "score": 83.04880688243107 }, "noul": { "abstention_rate": 0.17307692307692313, "brier": 0.10655681075571934, "ece": 0.06711648937487673, "n": 364, "score": 86.57670212502465 }, "score": { "interval90_coverage": 0.9402173913043478, "interval90_width": 0.263586956521739, "n": 368, "nrps": 0.07024138314041055, "score": 86.42387131465975, "top_ece": 0.10064059528319728 } }, "score": 85.34979344070516 }, "capability": 73.45697484423279, "composite_ci95": { "A": [ 21.062859904802277, 21.88568498833731 ], "B": [ 21.239170136076478, 22.480828485144727 ], "C": [ 18.819765630154432, 21.88568498833731 ] }, "cost": { "basis": "ESTIMATE (*est.), base-model market reference. The declared base Qwen/Qwen3.8-27B resolves in the frozen 25 Sep 2026 snapshot through pricing_v15 BASE_REFERENCES to qwen/qwen3.8-27b at USD 0.42 per 1M input and USD 3.00 per 1M output (verified before the pod was rented; r75 COST-BASIS.json). This row reports a real output token count - 1 token per readout, 1,471 in total - so the Gutsy zero-output rule does NOT apply and the output price is charged as measured. This row's own usage on draw v1.6-regular-20261010-a2: 694,381 input and 1,471 output tokens over 1,500 decisions.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.19736867999999996 }, "display": "RSI-Jev v6.1-VL 27B", "endpoint_condition": "evaluator-owned Lium H100 80GB pod (36 min, USD 1.95/h), sealed per chief-of-staff decision #16415 on Florian's YES: the author's own serve.cli inside a loopback-only network namespace (unshare -n; DNS and outbound TCP both proven dead from inside the run namespace), started with env -i and OPENAI_API_KEY empty, plus a pod-level iptables egress cut installed and verified before the sealed input was uploaded. HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE set only to block the download fallbacks the r75 review found; no source line was edited. Server pointed at the LOCAL PINNED SNAPSHOT DIRECTORY, not a repo id, so no revision could resolve to main. Unchanged defaults, --effort auto, 32,768-token cap, no truncation. 54,047 MiB of 81,559 MiB used.", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 68.48613704048307, "I_sealed": 54.64217545503776, "base": 61.564156247760415, "excess": 3.7707170041056006, "gap": 13.843961585445314, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 75.8256733782429, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 92.21621621621622, "hard": 73.57659641396666, "judge": 72.31377719182598, "standard": 77.39640016743407 } }, "open|noul": { "cc": 66.15458937198068, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 82.6086956521739, "judge": 39.99999999999999, "standard": 55.55555555555556 } }, "open|score": { "cc": 63.478148371225664, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 67.937072018553, "hard": 66.44443112298649, "judge": 61.06082661163181, "standard": 58.942103683431114 } }, "sealed|choice": { "cc": 70.33653729047333, "n": { "easy": 72, "hard": 260, "judge": 141, "standard": 127 }, "tiers": { "easy": 88.73709458754823, "hard": 64.87853138397273, "judge": 71.69093399624576, "standard": 70.02067539627843 } }, "sealed|noul": { "cc": 41.66298613667034, "n": { "easy": 39, "hard": 132, "judge": 72, "standard": 57 }, "tiers": { "easy": 58.974358974358964, "hard": 21.212121212121215, "judge": 50.0, "standard": 61.403508771929815 } }, "sealed|score": { "cc": 51.92700293796962, "n": { "easy": 33, "hard": 115, "judge": 75, "standard": 77 }, "tiers": { "easy": 68.04656098024147, "hard": 33.730600016105896, "judge": 64.21257035928524, "standard": 61.83167862858772 } } } }, "jevbench_score": 21.58850717988564, "key": "rsi-jev-v6.1-vl-27b", "last_measured_on": "2026-10-10", "licence": "Apache-2.0 (model card); source Shanghua-Gao/RSI-Jev@fc982bbd MIT", "listing": "ranked", "measured_in": "v1.6.5", "measurement_date_status": "measured", "model_pin": "shgao/rsi-jev-v6.1-vl-27b@d0dccc052ee844ab7481ddfacad1b7335ab394e0", "noul_decisive": { "acc_among_decisive": 0.9102990033222591, "decisive": 301, "decisive_rate": 0.8269230769230769, "n": 375, "supported": true, "valid": 364 }, "planned_weight_denominator": 1.0, "rank": 4, "ranked": true, "ranks": { "A": 4, "B": 4, "C": 3, "capability": 1 }, "repo": "https://huggingface.co/shgao/rsi-jev-v6.1-vl-27b", "scores": { "A": 21.58850717988564, "B": 22.011280090800952, "C": 21.58850717988564 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.39549588703666816, "p50_s_raw": 0.12274794351833407, "p95_s_adjusted": 0.8540048229158855, "p95_s_raw": 0.35200241145794275 }, "status": { "answered_ok": 1471, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns": { "family": { "calibration": { "accuracy": 0.9310344827586207, "cc": 86.4125424016998, "cc_by_type": { "choice": 100.0, "noul": 66.66666666666667, "score": 92.5709605384327 }, "low_n": true, "n": 29, "n_by_type": { "choice": 16, "noul": 6, "score": 7 }, "score": 86.4125424016998 }, "calibration_gen": { "accuracy": 0.7232142857142857, "cc": 58.19506668349169, "cc_by_type": { "choice": 67.3913043478261, "noul": 61.53846153846154, "score": 45.655434164187426 }, "low_n": false, "n": 112, "n_by_type": { "choice": 56, "noul": 26, "score": 30 }, "score": 58.19506668349169 }, "composite_gen": { "accuracy": 0.8877551020408163, "cc": 82.46075900003056, "cc_by_type": { "choice": 96.29629629629629, "noul": 78.57142857142858, "score": 72.51455213236682 }, "low_n": false, "n": 98, "n_by_type": { "choice": 36, "noul": 28, "score": 34 }, "score": 82.46075900003056 }, "fairness": { "accuracy": 0.9629629629629629, "cc": 94.1963028792578, "cc_by_type": { "choice": 88.23529411764707, "noul": 100.0, "score": 94.35361452012633 }, "low_n": true, "n": 27, "n_by_type": { "choice": 12, "noul": 6, "score": 9 }, "score": 94.1963028792578 }, "jaggedness": { "accuracy": 0.5081081081081081, "cc": 13.570657148702269, "cc_by_type": { "choice": 38.86058360352014, "noul": -48.71794871794872, "score": 50.56933656053539 }, "low_n": false, "n": 185, "n_by_type": { "choice": 97, "noul": 39, "score": 49 }, "score": 13.570657148702269 }, "llm_router": { "accuracy": 0.8598484848484849, "cc": 81.63698812259595, "cc_by_type": { "choice": 75.84765443567116, "noul": 86.30136986301369, "score": 82.76194006910303 }, "low_n": false, "n": 264, "n_by_type": { "choice": 131, "noul": 73, "score": 60 }, "score": 81.63698812259595 }, "long_state": { "accuracy": 0.0, "cc": -110.31746031746032, "cc_by_type": { "choice": -30.952380952380945, "noul": -100.0, "score": -200.0 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 11, "score": 7 }, "score": 0.0 }, "ml_usecase": { "accuracy": 0.6774193548387096, "cc": 31.38414817065086, "cc_by_type": { "choice": 72.30769230769232, "noul": 0.0, "score": 21.844752204260253 }, "low_n": false, "n": 31, "n_by_type": { "choice": 15, "noul": 8, "score": 8 }, "score": 31.38414817065086 }, "multilingual": { "accuracy": 0.7428571428571429, "cc": 58.6257780124704, "cc_by_type": { "choice": 70.44334975369458, "noul": 51.72413793103448, "score": 53.70984635268214 }, "low_n": false, "n": 315, "n_by_type": { "choice": 146, "noul": 87, "score": 82 }, "score": 58.6257780124704 }, "negation_pair_gen": { "accuracy": 0.875, "cc": 63.20741226657981, "cc_by_type": { "choice": 90.47619047619048, "noul": 60.00000000000001, "score": 39.146046323548944 }, "low_n": false, "n": 56, "n_by_type": { "choice": 28, "noul": 20, "score": 8 }, "score": 63.20741226657981 }, "option_order": { "accuracy": 0.9824561403508771, "cc": 97.78024417314094, "cc_by_type": { "choice": 97.78024417314094 }, "low_n": false, "n": 57, "n_by_type": { "choice": 57 }, "score": 97.78024417314094 }, "perturb_pair": { "accuracy": 0.9, "cc": 89.49429032954704, "cc_by_type": { "choice": 84.27260812581913, "noul": 100.0, "score": 84.21026286282202 }, "low_n": false, "n": 60, "n_by_type": { "choice": 34, "noul": 14, "score": 12 }, "score": 89.49429032954704 }, "usecase": { "accuracy": 0.7046413502109705, "cc": 51.83295324947296, "cc_by_type": { "choice": 73.40528040084796, "noul": 36.8421052631579, "score": 45.251474084413026 }, "low_n": false, "n": 237, "n_by_type": { "choice": 111, "noul": 57, "score": 69 }, "score": 51.83295324947296 } }, "lang": { "ar": { "accuracy": 0.8181818181818182, "cc": 69.11348259490397, "cc_by_type": { "choice": 73.33333333333334, "noul": 50.0, "score": 84.00711445137858 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 4, "score": 2 }, "score": 69.11348259490397 }, "cs": { "accuracy": 0.875, "cc": 87.28813559322035, "cc_by_type": { "choice": 74.57627118644068, "noul": 100.0 }, "low_n": true, "n": 8, "n_by_type": { "choice": 5, "noul": 3 }, "score": 87.28813559322035 }, "da": { "accuracy": 0.6666666666666666, "cc": 44.00477315138941, "cc_by_type": { "choice": 52.662721893491124, "noul": 19.999999999999996, "score": 59.351597560677114 }, "low_n": true, "n": 24, "n_by_type": { "choice": 11, "noul": 5, "score": 8 }, "score": 44.00477315138941 }, "de": { "accuracy": 0.8444444444444444, "cc": 72.3799653014141, "cc_by_type": { "choice": 83.70549185274592, "noul": 77.77777777777777, "score": 55.656626273718615 }, "low_n": false, "n": 45, "n_by_type": { "choice": 24, "noul": 9, "score": 12 }, "score": 72.3799653014141 }, "el": { "accuracy": 0.6666666666666666, "cc": -0.8800661948371129, "cc_by_type": { "choice": 75.90361445783132, "noul": -19.999999999999996, "score": -58.54381304234266 }, "low_n": true, "n": 18, "n_by_type": { "choice": 11, "noul": 5, "score": 2 }, "score": 0.0 }, "en": { "accuracy": 0.7497765862377123, "cc": 57.06152962458927, "cc_by_type": { "choice": 72.18171620863711, "noul": 42.85714285714286, "score": 56.14572980798784 }, "low_n": false, "n": 1119, "n_by_type": { "choice": 577, "noul": 266, "score": 276 }, "score": 57.06152962458927 }, "es": { "accuracy": 0.8928571428571429, "cc": 82.61244146186118, "cc_by_type": { "choice": 86.90909090909093, "noul": 100.0, "score": 60.9282334764926 }, "low_n": true, "n": 28, "n_by_type": { "choice": 10, "noul": 11, "score": 7 }, "score": 82.61244146186118 }, "fi": { "accuracy": 0.5555555555555556, "cc": 34.97652582159624, "cc_by_type": { "choice": 36.61971830985915, "noul": 33.33333333333333 }, "low_n": true, "n": 9, "n_by_type": { "choice": 6, "noul": 3 }, "score": 34.97652582159624 }, "fr": { "accuracy": 0.6666666666666666, "cc": 41.96733456321797, "cc_by_type": { "choice": 74.61212976022568, "noul": 0.0, "score": 51.28987392942821 }, "low_n": true, "n": 15, "n_by_type": { "choice": 5, "noul": 4, "score": 6 }, "score": 41.96733456321797 }, "hi": { "accuracy": 0.6666666666666666, "cc": 52.85728930896761, "cc_by_type": { "choice": 62.73291925465839, "noul": 60.00000000000001, "score": 35.83894867224444 }, "low_n": false, "n": 30, "n_by_type": { "choice": 11, "noul": 10, "score": 9 }, "score": 52.85728930896761 }, "id": { "accuracy": 0.8333333333333334, "cc": 83.87096774193549, "cc_by_type": { "choice": 67.74193548387096, "noul": 100.0 }, "low_n": true, "n": 6, "n_by_type": { "choice": 4, "noul": 2 }, "score": 83.87096774193549 }, "it": { "accuracy": 1.0, "cc": 90.29663262626855, "cc_by_type": { "choice": 100.00000000000001, "noul": 100.0, "score": 70.88989787880564 }, "low_n": true, "n": 17, "n_by_type": { "choice": 5, "noul": 4, "score": 8 }, "score": 90.29663262626855 }, "ja": { "accuracy": 0.53125, "cc": 32.05622061795162, "cc_by_type": { "choice": 43.08943089430895, "noul": 14.28571428571428, "score": 38.79351667383163 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 32.05622061795162 }, "ko": { "accuracy": 0.8181818181818182, "cc": 85.45781546660389, "cc_by_type": { "choice": 100.0, "noul": 100.0, "score": 56.373446399811655 }, "low_n": true, "n": 11, "n_by_type": { "choice": 4, "noul": 2, "score": 5 }, "score": 85.45781546660389 }, "mixed": { "accuracy": 1.0, "cc": 99.9689563002903, "cc_by_type": { "choice": 100.0, "noul": 100.0, "score": 99.9068689008709 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 4, "score": 1 }, "score": 99.9689563002903 }, "nl": { "accuracy": 0.6923076923076923, "cc": 40.991198714143415, "cc_by_type": { "choice": 74.93036211699165, "noul": 33.33333333333333, "score": 14.709900692105283 }, "low_n": true, "n": 13, "n_by_type": { "choice": 5, "noul": 6, "score": 2 }, "score": 40.991198714143415 }, "no": { "accuracy": 0.7142857142857143, "cc": 67.21311475409837, "cc_by_type": { "choice": 34.42622950819673, "noul": 100.0 }, "low_n": true, "n": 7, "n_by_type": { "choice": 4, "noul": 3 }, "score": 67.21311475409837 }, "pl": { "accuracy": 0.7, "cc": 51.32398893140217, "cc_by_type": { "choice": 92.12942719720158, "noul": 19.999999999999996, "score": 41.84253959700492 }, "low_n": false, "n": 40, "n_by_type": { "choice": 16, "noul": 10, "score": 14 }, "score": 51.32398893140217 }, "pt": { "accuracy": 0.8333333333333334, "cc": 83.10065347581066, "cc_by_type": { "choice": 63.81909547738693, "noul": 100.0, "score": 85.48286495004504 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 6, "score": 5 }, "score": 83.10065347581066 }, "sv": { "accuracy": 0.625, "cc": 53.29148096066934, "cc_by_type": { "choice": 11.111111111111109, "noul": 50.0, "score": 98.76333177089691 }, "low_n": true, "n": 8, "n_by_type": { "choice": 3, "noul": 4, "score": 1 }, "score": 53.29148096066934 }, "tr": { "accuracy": 0.7272727272727273, "cc": 23.85319314395998, "cc_by_type": { "choice": 73.6842105263158, "noul": 33.33333333333333, "score": -35.457964427769184 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 3, "score": 3 }, "score": 23.85319314395998 }, "uk": { "accuracy": 0.5454545454545454, "cc": 4.1198605058986955, "cc_by_type": { "choice": 34.78260869565218, "noul": -100.0, "score": 77.5769728220439 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 1, "score": 4 }, "score": 4.1198605058986955 }, "zh": { "accuracy": 0.875, "cc": 73.56783818557031, "cc_by_type": { "choice": 100.00000000000001, "noul": 33.33333333333333, "score": 87.37018122337759 }, "low_n": true, "n": 8, "n_by_type": { "choice": 4, "noul": 3, "score": 1 }, "score": 73.56783818557031 } }, "type": { "choice": { "accuracy": 0.7826666666666666, "cc": 71.84315080324754, "cc_by_type": { "choice": 71.84315080324754 }, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 71.84315080324754 }, "noul": { "accuracy": 0.7306666666666667, "cc": 46.13333333333334, "cc_by_type": { "noul": 46.13333333333334 }, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 46.13333333333334 }, "score": { "accuracy": 0.6986666666666667, "cc": 54.531190100580204, "cc_by_type": { "score": 54.531190100580204 }, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 54.531190100580204 } }, "usecase": { "advertising": { "accuracy": 0.7083333333333334, "cc": 36.966773340637815, "cc_by_type": { "choice": 70.34596375617794, "noul": -19.999999999999996, "score": 60.55435626573551 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 36.966773340637815 }, "code_linting": { "accuracy": 0.75, "cc": 68.95860502475776, "cc_by_type": { "choice": 70.54009819967267, "noul": 100.0, "score": 36.33571687460061 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 68.95860502475776 }, "customer_support": { "accuracy": 0.75, "cc": 47.499513085946944, "cc_by_type": { "choice": 67.74193548387096, "noul": 75.0, "score": -0.24339622603013478 }, "low_n": true, "n": 20, "n_by_type": { "choice": 8, "noul": 8, "score": 4 }, "score": 47.499513085946944 }, "demand_forecasting": { "accuracy": 0.3076923076923077, "cc": -17.782585557336656, "cc_by_type": { "choice": 15.662650602409636, "noul": -50.0, "score": -19.010407274419606 }, "low_n": true, "n": 26, "n_by_type": { "choice": 11, "noul": 4, "score": 11 }, "score": 0.0 }, "ecommerce_marketplaces": { "accuracy": 0.5555555555555556, "cc": 28.57990326207394, "cc_by_type": { "choice": 62.73291925465839, "noul": -19.999999999999996, "score": 43.006790531563425 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 5, "score": 6 }, "score": 28.57990326207394 }, "feature_extraction": { "accuracy": 0.8620689655172413, "cc": 64.36666498139438, "cc_by_type": { "choice": 82.90598290598291, "noul": 19.999999999999996, "score": 90.19401203820024 }, "low_n": true, "n": 29, "n_by_type": { "choice": 15, "noul": 5, "score": 9 }, "score": 64.36666498139438 }, "financial_crime": { "accuracy": 0.7692307692307693, "cc": 69.24951206795548, "cc_by_type": { "choice": 86.95652173913045, "noul": 100.0, "score": 20.792014464735985 }, "low_n": true, "n": 26, "n_by_type": { "choice": 10, "noul": 8, "score": 8 }, "score": 69.24951206795548 }, "gaming": { "accuracy": 0.5862068965517241, "cc": 25.37172392511331, "cc_by_type": { "choice": 52.475247524752476, "noul": -14.28571428571429, "score": 37.92563853630174 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 7, "score": 11 }, "score": 25.37172392511331 }, "insurance_claims": { "accuracy": 0.5714285714285714, "cc": 31.669815093397148, "cc_by_type": { "choice": 52.38095238095238, "noul": -19.999999999999996, "score": 62.62849289923906 }, "low_n": true, "n": 14, "n_by_type": { "choice": 8, "noul": 5, "score": 1 }, "score": 31.669815093397148 }, "knowledge_graphs": { "accuracy": 0.6956521739130435, "cc": 42.704203038079555, "cc_by_type": { "choice": 72.51908396946564, "noul": 19.999999999999996, "score": 35.59352514477303 }, "low_n": true, "n": 23, "n_by_type": { "choice": 14, "noul": 5, "score": 4 }, "score": 42.704203038079555 }, "lead_generation": { "accuracy": 0.72, "cc": 55.47104563667543, "cc_by_type": { "choice": 100.0, "noul": 25.0, "score": 41.413136910026275 }, "low_n": true, "n": 25, "n_by_type": { "choice": 8, "noul": 8, "score": 9 }, "score": 55.47104563667543 }, "legal_compliance": { "accuracy": 0.6923076923076923, "cc": 8.840061259572431, "cc_by_type": { "choice": 67.21311475409836, "noul": -100.0, "score": 59.30706902461893 }, "low_n": true, "n": 13, "n_by_type": { "choice": 8, "noul": 1, "score": 4 }, "score": 8.840061259572431 }, "llm_guardrails": { "accuracy": 0.9310344827586207, "cc": 89.90702297249, "cc_by_type": { "choice": 90.8256880733945, "noul": 100.0, "score": 78.89538084407552 }, "low_n": true, "n": 29, "n_by_type": { "choice": 14, "noul": 8, "score": 7 }, "score": 89.90702297249 }, "model_routing": { "accuracy": 0.86, "cc": 76.7397155888396, "cc_by_type": { "choice": 85.52666845349772, "noul": 81.81818181818181, "score": 62.874296494839264 }, "low_n": false, "n": 150, "n_by_type": { "choice": 74, "noul": 44, "score": 32 }, "score": 76.7397155888396 }, "moderation_trust_safety": { "accuracy": 0.6875, "cc": 52.07409272558595, "cc_by_type": { "choice": 52.63157894736842, "noul": 60.00000000000001, "score": 43.590699229389415 }, "low_n": true, "n": 16, "n_by_type": { "choice": 8, "noul": 5, "score": 3 }, "score": 52.07409272558595 }, "recruiting": { "accuracy": 0.6666666666666666, "cc": 54.013036775459085, "cc_by_type": { "choice": 54.77386934673366, "noul": 66.66666666666667, "score": 40.59857431297692 }, "low_n": true, "n": 21, "n_by_type": { "choice": 9, "noul": 6, "score": 6 }, "score": 54.013036775459085 }, "risk_assessment": { "accuracy": 0.6153846153846154, "cc": 46.84943347419826, "cc_by_type": { "choice": 62.264150943396224, "noul": 33.33333333333333, "score": 44.95081614586523 }, "low_n": true, "n": 13, "n_by_type": { "choice": 7, "noul": 3, "score": 3 }, "score": 46.84943347419826 }, "scientific_discovery": { "accuracy": 0.59375, "cc": 25.763079128747716, "cc_by_type": { "choice": 60.1593625498008, "noul": -14.28571428571429, "score": 31.41558912215664 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 25.763079128747716 }, "search_retrieval": { "accuracy": 0.9090909090909091, "cc": 86.99121578361938, "cc_by_type": { "choice": 100.0, "noul": 100.0, "score": 60.97364735085814 }, "low_n": true, "n": 22, "n_by_type": { "choice": 9, "noul": 6, "score": 7 }, "score": 86.99121578361938 } } }, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 80.64755807175129, 87.47437139642248 ], "I": [ 56.725930974421935, 65.23484456387472 ], "capability": [ 69.35974616373294, 75.75238339213846 ], "composite": { "A": [ 21.062859904802277, 21.88568498833731 ], "B": [ 21.239170136076478, 22.480828485144727 ], "C": [ 18.819765630154432, 21.88568498833731 ] }, "seed": 27 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": false, "g_med_used": 10.073244581339713, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 85.34979344070516, "capability": 73.45697484423279, "intelligence": 61.564156247760415 }, "per_type": { "choice": { "accuracy": 0.7826666666666666, "calibration": 83.04880688243107, "cc": 71.84315080324754, "cc_by_type": { "choice": 71.84315080324754 }, "cc_tiered": 73.08110533435811, "cc_tiered_open": 75.8256733782429, "cc_tiered_sealed": 70.33653729047333, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 71.84315080324754 }, "noul": { "accuracy": 0.7306666666666667, "calibration": 86.57670212502465, "cc": 46.13333333333334, "cc_by_type": { "noul": 46.13333333333334 }, "cc_tiered": 53.90878775432551, "cc_tiered_open": 66.15458937198068, "cc_tiered_sealed": 41.66298613667034, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 46.13333333333334 }, "score": { "accuracy": 0.6986666666666667, "calibration": 86.42387131465975, "cc": 54.531190100580204, "cc_by_type": { "score": 54.531190100580204 }, "cc_tiered": 57.70257565459764, "cc_tiered_open": 63.478148371225664, "cc_tiered_sealed": 51.92700293796962, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 54.531190100580204 } }, "raw_on_A": { "calibration": 84.95320234939237, "capability": 73.13630404134082, "intelligence": 61.31940573328927 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.014666666666666717, "n": 750 }, "noul": { "invalid_rate": 0.029333333333333322, "n": 375 }, "score": { "invalid_rate": 0.01866666666666672, "n": 375 } }, "views": { "Balanced 33:33:33": 19.344794057306224, "Emphasis on Accuracy 60:20:20": 20.935791627644683, "Emphasis on Cost 20:20:60": 15.593981583505036, "Emphasis on Speed 20:60:20": 23.15453324264071, "Intelligence only": 23.882030176915077 } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Standard Thinking", "axes": { "calibration": 85.49831885164681, "cost": 43.21124696592469, "intelligence": 34.464905061466695, "speed": 84.28099014807135 }, "calibration": { "parts": { "choice": { "ece_hard": 0.07091377459977057, "ece_score": 85.8172450800459, "mean_tvd": null, "n": 739, "n_ece_hard": 317, "n_tvd": 0, "score": 85.8172450800459 }, "noul": { "abstention_rate": 0.39010989010989006, "brier": 0.15382831780922446, "ece": 0.0674352050569083, "n": 364, "score": 86.51295898861834 }, "score": { "interval90_coverage": 0.9320652173913043, "interval90_width": 0.5106431159420289, "n": 368, "nrps": 0.11464419484173276, "score": 84.1647524862762, "top_ece": 0.10103037771637155 } }, "score": 85.49831885164681 }, "capability": 59.981611956556755, "composite_ci95": { "A": [ 12.633047380555162, 23.788181701570505 ], "B": [ 11.128195660262914, 21.916709714788084 ], "C": [ 8.772949569829972, 16.519570626090626 ] }, "cost": { "basis": "v1.5-M2 base-model reference: model card base_model StandardThinking/StandardOne-8B, built on mistralai/Ministral-3-8B-Instruct-2512 (not listed on OpenRouter as Standard One). OpenRouter mistralai/ministral-8b-2512 list price USD 0.15/M input and USD 0.15/M output, read 2026-10-10T08:43Z from https://openrouter.ai/api/v1/models. Schema head generates no text: 0 output tokens.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0781553 }, "display": "Standard One 8B SH (Standard Thinking)", "endpoint_condition": "evaluator-owned Lium GPU pod (H100 80GB), offline read-only container (--network none); author's reference server code/serve_head.py, unchanged defaults", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 42.65983609411336, "I_sealed": 26.269974028820027, "base": 34.464905061466695, "excess": 6.316617483953619, "gap": 16.389862065293332, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 64.76188322786868, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 88.32432432432432, "hard": 56.59012268008808, "judge": 64.40342781806197, "standard": 69.8618668899121 } }, "open|noul": { "cc": 26.588681849551417, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": 39.13043478260869, "judge": 10.000000000000009, "standard": 11.111111111111116 } }, "open|score": { "cc": 36.628943204920006, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 20.91980977072071, "hard": 40.258980287858556, "judge": 46.927298157653716, "standard": 21.775903327041945 } }, "sealed|choice": { "cc": 53.894778364676796, "n": { "easy": 72, "hard": 260, "judge": 141, "standard": 127 }, "tiers": { "easy": 60.57983105641882, "hard": 50.5331427943278, "judge": 51.60062909035564, "standard": 60.716747070985534 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 39, "hard": 132, "judge": 72, "standard": 57 }, "tiers": { "easy": 12.82051282051282, "hard": -9.090909090909093, "judge": 11.111111111111116, "standard": -8.771929824561408 } }, "sealed|score": { "cc": 24.915143721783284, "n": { "easy": 33, "hard": 115, "judge": 75, "standard": 77 }, "tiers": { "easy": 48.44464458742374, "hard": 10.686057111566038, "judge": 28.029772063789803, "standard": 36.93662399638777 } } } }, "jevbench_score": 18.74677558938401, "key": "standard_one_8b_sh", "last_measured_on": "2026-10-10", "licence": "Apache-2.0 (merged weights, schema head and reference server; base Ministral 3 8B Apache-2.0)", "listing": "ranked", "measured_in": "v1.6.5", "measurement_date_status": "measured", "model_pin": "StandardThinking/StandardOne-8B-SH@25e5739bd9635b54f925cf9ba2daa2ec42afd19e", "noul_decisive": { "acc_among_decisive": 0.8828828828828829, "decisive": 222, "decisive_rate": 0.6098901098901099, "n": 375, "supported": true, "valid": 364 }, "planned_weight_denominator": 1.0, "rank": 5, "ranked": true, "ranks": { "A": 5, "B": 5, "C": 5, "capability": 5 }, "repo": "https://huggingface.co/StandardThinking/StandardOne-8B-SH", "scores": { "A": 18.74677558938401, "B": 16.94153091767041, "C": 13.018594159294452 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.27213579495437445, "p50_s_raw": 0.061067897477187216, "p95_s_adjusted": 1.3712458158901424, "p95_s_raw": 0.6106229079450712 }, "status": { "answered_ok": 1471, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns": { "family": { "calibration": { "accuracy": 0.896551724137931, "cc": 76.24972247673712, "cc_by_type": { "choice": 83.33333333333333, "noul": 100.0, "score": 45.41583409687805 }, "low_n": true, "n": 29, "n_by_type": { "choice": 16, "noul": 6, "score": 7 }, "score": 76.24972247673712 }, "calibration_gen": { "accuracy": 0.6339285714285714, "cc": 40.6073873879211, "cc_by_type": { "choice": 60.40372670807453, "noul": 30.76923076923077, "score": 30.64920468645799 }, "low_n": false, "n": 112, "n_by_type": { "choice": 56, "noul": 26, "score": 30 }, "score": 40.6073873879211 }, "composite_gen": { "accuracy": 0.6428571428571429, "cc": 42.991452161771214, "cc_by_type": { "choice": 70.37037037037037, "noul": 28.57142857142858, "score": 30.032557543514706 }, "low_n": false, "n": 98, "n_by_type": { "choice": 36, "noul": 28, "score": 34 }, "score": 42.991452161771214 }, "fairness": { "accuracy": 0.9629629629629629, "cc": 88.7254774240554, "cc_by_type": { "choice": 100.0, "noul": 100.0, "score": 66.17643227216622 }, "low_n": true, "n": 27, "n_by_type": { "choice": 12, "noul": 6, "score": 9 }, "score": 88.7254774240554 }, "jaggedness": { "accuracy": 0.4702702702702703, "cc": 8.921712067552678, "cc_by_type": { "choice": 34.69198703103288, "noul": -53.84615384615385, "score": 45.919303017779 }, "low_n": false, "n": 185, "n_by_type": { "choice": 97, "noul": 39, "score": 49 }, "score": 8.921712067552678 }, "llm_router": { "accuracy": 0.6477272727272727, "cc": 47.79185607418858, "cc_by_type": { "choice": 57.268927078495125, "noul": 26.027397260273965, "score": 60.07924388379662 }, "low_n": false, "n": 264, "n_by_type": { "choice": 131, "noul": 73, "score": 60 }, "score": 47.79185607418858 }, "long_state": { "accuracy": 0.0, "cc": -110.31746031746032, "cc_by_type": { "choice": -30.952380952380945, "noul": -100.0, "score": -200.0 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 11, "score": 7 }, "score": 0.0 }, "ml_usecase": { "accuracy": 0.3225806451612903, "cc": -19.288646333408973, "cc_by_type": { "choice": 26.153846153846153, "noul": -75.0, "score": -9.019785154073068 }, "low_n": false, "n": 31, "n_by_type": { "choice": 15, "noul": 8, "score": 8 }, "score": 0.0 }, "multilingual": { "accuracy": 0.5873015873015873, "cc": 28.891423678941237, "cc_by_type": { "choice": 56.112852664576806, "noul": 8.045977011494255, "score": 22.515441360752643 }, "low_n": false, "n": 315, "n_by_type": { "choice": 146, "noul": 87, "score": 82 }, "score": 28.891423678941237 }, "negation_pair_gen": { "accuracy": 0.5357142857142857, "cc": -27.351155376155678, "cc_by_type": { "choice": 76.19047619047619, "noul": -50.0, "score": -108.24394231894323 }, "low_n": false, "n": 56, "n_by_type": { "choice": 28, "noul": 20, "score": 8 }, "score": 0.0 }, "option_order": { "accuracy": 0.9122807017543859, "cc": 88.90122086570477, "cc_by_type": { "choice": 88.90122086570477 }, "low_n": false, "n": 57, "n_by_type": { "choice": 57 }, "score": 88.90122086570477 }, "perturb_pair": { "accuracy": 0.8333333333333334, "cc": 73.07758294752922, "cc_by_type": { "choice": 76.4089121887287, "noul": 71.42857142857142, "score": 71.39526522528755 }, "low_n": false, "n": 60, "n_by_type": { "choice": 34, "noul": 14, "score": 12 }, "score": 73.07758294752922 }, "usecase": { "accuracy": 0.4978902953586498, "cc": 20.903453186109534, "cc_by_type": { "choice": 45.65426864521104, "noul": 1.7543859649122862, "score": 15.301704948205275 }, "low_n": false, "n": 237, "n_by_type": { "choice": 111, "noul": 57, "score": 69 }, "score": 20.903453186109534 } }, "lang": { "ar": { "accuracy": 0.2727272727272727, "cc": -41.39521348113611, "cc_by_type": { "choice": -6.666666666666665, "noul": 0.0, "score": -117.51897377674166 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 4, "score": 2 }, "score": 0.0 }, "cs": { "accuracy": 0.375, "cc": -4.802259887005647, "cc_by_type": { "choice": 23.72881355932204, "noul": -33.333333333333336 }, "low_n": true, "n": 8, "n_by_type": { "choice": 5, "noul": 3 }, "score": 0.0 }, "da": { "accuracy": 0.5833333333333334, "cc": 23.301107918011525, "cc_by_type": { "choice": 28.994082840236683, "noul": -19.999999999999996, "score": 60.90924091379788 }, "low_n": true, "n": 24, "n_by_type": { "choice": 11, "noul": 5, "score": 8 }, "score": 23.301107918011525 }, "de": { "accuracy": 0.6444444444444445, "cc": 44.56362261696139, "cc_by_type": { "choice": 72.84248642124321, "noul": 55.55555555555556, "score": 5.292825874085382 }, "low_n": false, "n": 45, "n_by_type": { "choice": 24, "noul": 9, "score": 12 }, "score": 44.56362261696139 }, "el": { "accuracy": 0.5, "cc": -22.37065677373447, "cc_by_type": { "choice": 63.855421686746986, "noul": -60.0, "score": -70.96739200795041 }, "low_n": true, "n": 18, "n_by_type": { "choice": 11, "noul": 5, "score": 2 }, "score": 0.0 }, "en": { "accuracy": 0.5978552278820375, "cc": 30.293789733525, "cc_by_type": { "choice": 56.70218732473357, "noul": 3.007518796992481, "score": 31.171663078848944 }, "low_n": false, "n": 1119, "n_by_type": { "choice": 577, "noul": 266, "score": 276 }, "score": 30.293789733525 }, "es": { "accuracy": 0.7142857142857143, "cc": 43.83935759486882, "cc_by_type": { "choice": 100.0, "noul": 45.45454545454546, "score": -13.936472669939004 }, "low_n": true, "n": 28, "n_by_type": { "choice": 10, "noul": 11, "score": 7 }, "score": 43.83935759486882 }, "fi": { "accuracy": 0.2222222222222222, "cc": -42.25352112676057, "cc_by_type": { "choice": 15.49295774647887, "noul": -100.0 }, "low_n": true, "n": 9, "n_by_type": { "choice": 6, "noul": 3 }, "score": 0.0 }, "fr": { "accuracy": 0.4666666666666667, "cc": -0.26931923616449654, "cc_by_type": { "choice": 49.22425952045134, "noul": -100.0, "score": 49.96778277105517 }, "low_n": true, "n": 15, "n_by_type": { "choice": 5, "noul": 4, "score": 6 }, "score": 0.0 }, "hi": { "accuracy": 0.4, "cc": 3.9605189196118, "cc_by_type": { "choice": 25.465838509316765, "noul": -19.999999999999996, "score": 6.415718249518632 }, "low_n": false, "n": 30, "n_by_type": { "choice": 11, "noul": 10, "score": 9 }, "score": 3.9605189196118 }, "id": { "accuracy": 0.5, "cc": -16.12903225806452, "cc_by_type": { "choice": 67.74193548387096, "noul": -100.0 }, "low_n": true, "n": 6, "n_by_type": { "choice": 4, "noul": 2 }, "score": 0.0 }, "it": { "accuracy": 0.7647058823529411, "cc": 42.698737038013526, "cc_by_type": { "choice": 100.00000000000001, "noul": 0.0, "score": 28.09621111404058 }, "low_n": true, "n": 17, "n_by_type": { "choice": 5, "noul": 4, "score": 8 }, "score": 42.698737038013526 }, "ja": { "accuracy": 0.5, "cc": 9.04085560182647, "cc_by_type": { "choice": 34.959349593495936, "noul": -42.85714285714286, "score": 35.02036006912633 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 9.04085560182647 }, "ko": { "accuracy": 0.6363636363636364, "cc": 51.84513737928095, "cc_by_type": { "choice": 67.21311475409836, "noul": 100.0, "score": -11.677702616255514 }, "low_n": true, "n": 11, "n_by_type": { "choice": 4, "noul": 2, "score": 5 }, "score": 51.84513737928095 }, "mixed": { "accuracy": 1.0, "cc": 95.1901995158086, "cc_by_type": { "choice": 100.0, "noul": 100.0, "score": 85.57059854742577 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 4, "score": 1 }, "score": 95.1901995158086 }, "nl": { "accuracy": 0.46153846153846156, "cc": -6.51647728738407, "cc_by_type": { "choice": 49.860724233983284, "noul": 0.0, "score": -69.4101560961355 }, "low_n": true, "n": 13, "n_by_type": { "choice": 5, "noul": 6, "score": 2 }, "score": 0.0 }, "no": { "accuracy": 0.42857142857142855, "cc": 17.486338797814206, "cc_by_type": { "choice": 1.6393442622950836, "noul": 33.33333333333333 }, "low_n": true, "n": 7, "n_by_type": { "choice": 4, "noul": 3 }, "score": 17.486338797814206 }, "pl": { "accuracy": 0.575, "cc": 41.28299201238961, "cc_by_type": { "choice": 76.38828159160472, "noul": 19.999999999999996, "score": 27.460694445564116 }, "low_n": false, "n": 40, "n_by_type": { "choice": 16, "noul": 10, "score": 14 }, "score": 41.28299201238961 }, "pt": { "accuracy": 0.8333333333333334, "cc": 76.5780001477811, "cc_by_type": { "choice": 81.90954773869345, "noul": 100.0, "score": 47.82445270464984 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 6, "score": 5 }, "score": 76.5780001477811 }, "sv": { "accuracy": 0.5, "cc": 6.320693289140117, "cc_by_type": { "choice": 11.111111111111109, "noul": 50.0, "score": -42.14903124369076 }, "low_n": true, "n": 8, "n_by_type": { "choice": 3, "noul": 4, "score": 1 }, "score": 6.320693289140117 }, "tr": { "accuracy": 0.6363636363636364, "cc": 22.124025198251957, "cc_by_type": { "choice": 73.6842105263158, "noul": 33.33333333333333, "score": -40.64546826489326 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 3, "score": 3 }, "score": 22.124025198251957 }, "uk": { "accuracy": 0.5454545454545454, "cc": -5.12371977540209, "cc_by_type": { "choice": 34.78260869565218, "noul": -100.0, "score": 49.84623197814155 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 1, "score": 4 }, "score": 0.0 }, "zh": { "accuracy": 0.875, "cc": 62.994622040580985, "cc_by_type": { "choice": 100.00000000000001, "noul": 33.33333333333333, "score": 55.65053278840959 }, "low_n": true, "n": 8, "n_by_type": { "choice": 4, "noul": 3, "score": 1 }, "score": 62.994622040580985 } }, "type": { "choice": { "accuracy": 0.6653333333333333, "cc": 56.64190706512352, "cc_by_type": { "choice": 56.64190706512352 }, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 56.64190706512352 }, "noul": { "accuracy": 0.5226666666666666, "cc": 4.533333333333323, "cc_by_type": { "noul": 4.533333333333323 }, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 4.533333333333323 }, "score": { "accuracy": 0.5173333333333333, "cc": 27.98616472792205, "cc_by_type": { "score": 27.98616472792205 }, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 27.98616472792205 } }, "usecase": { "advertising": { "accuracy": 0.5833333333333334, "cc": 2.1721868800764432, "cc_by_type": { "choice": 50.57660626029654, "noul": -60.0, "score": 15.939954379932786 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 2.1721868800764432 }, "code_linting": { "accuracy": 0.2916666666666667, "cc": 4.537943129820239, "cc_by_type": { "choice": 1.8003273322422266, "noul": 60.00000000000001, "score": -48.186497942781514 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 4.537943129820239 }, "customer_support": { "accuracy": 0.5, "cc": 18.203665219669762, "cc_by_type": { "choice": 35.483870967741936, "noul": -25.0, "score": 44.12712469126735 }, "low_n": true, "n": 20, "n_by_type": { "choice": 8, "noul": 8, "score": 4 }, "score": 18.203665219669762 }, "demand_forecasting": { "accuracy": 0.3076923076923077, "cc": -15.30308720258734, "cc_by_type": { "choice": 27.710843373493972, "noul": -50.0, "score": -23.620104981255995 }, "low_n": true, "n": 26, "n_by_type": { "choice": 11, "noul": 4, "score": 11 }, "score": 0.0 }, "ecommerce_marketplaces": { "accuracy": 0.5555555555555556, "cc": 31.8743625129997, "cc_by_type": { "choice": 44.099378881987576, "noul": 19.999999999999996, "score": 31.523708657011518 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 5, "score": 6 }, "score": 31.8743625129997 }, "feature_extraction": { "accuracy": 0.5172413793103449, "cc": 16.864472729203204, "cc_by_type": { "choice": 31.623931623931618, "noul": -19.999999999999996, "score": 38.96948656367799 }, "low_n": true, "n": 29, "n_by_type": { "choice": 15, "noul": 5, "score": 9 }, "score": 16.864472729203204 }, "financial_crime": { "accuracy": 0.5, "cc": 15.494473495061596, "cc_by_type": { "choice": 34.78260869565218, "noul": 0.0, "score": 11.700811789532606 }, "low_n": true, "n": 26, "n_by_type": { "choice": 10, "noul": 8, "score": 8 }, "score": 15.494473495061596 }, "gaming": { "accuracy": 0.3793103448275862, "cc": -6.714417440674549, "cc_by_type": { "choice": 40.59405940594059, "noul": -42.85714285714286, "score": -17.88016887082138 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 7, "score": 11 }, "score": 0.0 }, "insurance_claims": { "accuracy": 0.5, "cc": 28.798707116861394, "cc_by_type": { "choice": 52.38095238095238, "noul": -60.0, "score": 94.0151689696318 }, "low_n": true, "n": 14, "n_by_type": { "choice": 8, "noul": 5, "score": 1 }, "score": 28.798707116861394 }, "knowledge_graphs": { "accuracy": 0.391304347826087, "cc": -0.31602635093334186, "cc_by_type": { "choice": 35.87786259541985, "noul": -60.0, "score": 23.174058351780126 }, "low_n": true, "n": 23, "n_by_type": { "choice": 14, "noul": 5, "score": 4 }, "score": 0.0 }, "lead_generation": { "accuracy": 0.36, "cc": 1.8570013967353607, "cc_by_type": { "choice": 83.73983739837398, "noul": -50.0, "score": -28.168833208167896 }, "low_n": true, "n": 25, "n_by_type": { "choice": 8, "noul": 8, "score": 9 }, "score": 1.8570013967353607 }, "legal_compliance": { "accuracy": 0.3076923076923077, "cc": -26.63894890640322, "cc_by_type": { "choice": 34.42622950819673, "noul": -100.0, "score": -14.343076227406382 }, "low_n": true, "n": 13, "n_by_type": { "choice": 8, "noul": 1, "score": 4 }, "score": 0.0 }, "llm_guardrails": { "accuracy": 0.7241379310344828, "cc": 51.33693366193592, "cc_by_type": { "choice": 72.4770642201835, "noul": 50.0, "score": 31.533736765624287 }, "low_n": true, "n": 29, "n_by_type": { "choice": 14, "noul": 8, "score": 7 }, "score": 51.33693366193592 }, "model_routing": { "accuracy": 0.68, "cc": 48.65467531366116, "cc_by_type": { "choice": 75.87778075582953, "noul": 36.36363636363635, "score": 33.72260882151758 }, "low_n": false, "n": 150, "n_by_type": { "choice": 74, "noul": 44, "score": 32 }, "score": 48.65467531366116 }, "moderation_trust_safety": { "accuracy": 0.5625, "cc": 25.43524834873031, "cc_by_type": { "choice": 36.84210526315789, "noul": 19.999999999999996, "score": 19.463639783033038 }, "low_n": true, "n": 16, "n_by_type": { "choice": 8, "noul": 5, "score": 3 }, "score": 25.43524834873031 }, "recruiting": { "accuracy": 0.6666666666666666, "cc": 52.73441114090357, "cc_by_type": { "choice": 84.92462311557789, "noul": 33.33333333333333, "score": 39.945276973799494 }, "low_n": true, "n": 21, "n_by_type": { "choice": 9, "noul": 6, "score": 6 }, "score": 52.73441114090357 }, "risk_assessment": { "accuracy": 0.23076923076923078, "cc": -34.30633768347406, "cc_by_type": { "choice": 5.660377358490565, "noul": -100.0, "score": -8.579390408912758 }, "low_n": true, "n": 13, "n_by_type": { "choice": 7, "noul": 3, "score": 3 }, "score": 0.0 }, "scientific_discovery": { "accuracy": 0.40625, "cc": -3.5610027209667976, "cc_by_type": { "choice": 36.254980079681275, "noul": -71.42857142857143, "score": 24.490583185989767 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 0.0 }, "search_retrieval": { "accuracy": 0.7272727272727273, "cc": 72.3432480209567, "cc_by_type": { "choice": 85.68019093078759, "noul": 100.0, "score": 31.349553132082498 }, "low_n": true, "n": 22, "n_by_type": { "choice": 9, "noul": 6, "score": 7 }, "score": 72.3432480209567 } } }, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 79.15231941930905, 86.38112582622072 ], "I": [ 29.324371200107983, 38.19996868954735 ], "capability": [ 55.13570575242618, 61.67190307298117 ], "composite": { "A": [ 12.633047380555162, 23.788181701570505 ], "B": [ 11.128195660262914, 21.916709714788084 ], "C": [ 8.772949569829972, 16.519570626090626 ] }, "seed": 28 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": false, "g_med_used": 10.073244581339713, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 85.49831885164681, "capability": 59.981611956556755, "intelligence": 34.464905061466695 }, "per_type": { "choice": { "accuracy": 0.6653333333333333, "calibration": 85.8172450800459, "cc": 56.64190706512352, "cc_by_type": { "choice": 56.64190706512352 }, "cc_tiered": 59.32833079627274, "cc_tiered_open": 64.76188322786868, "cc_tiered_sealed": 53.894778364676796, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 56.64190706512352 }, "noul": { "accuracy": 0.5226666666666666, "calibration": 86.51295898861834, "cc": 4.533333333333323, "cc_by_type": { "noul": 4.533333333333323 }, "cc_tiered": 13.294340924775709, "cc_tiered_open": 26.588681849551417, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 4.533333333333323 }, "score": { "accuracy": 0.5173333333333333, "calibration": 84.1647524862762, "cc": 27.98616472792205, "cc_by_type": { "score": 27.98616472792205 }, "cc_tiered": 30.772043463351643, "cc_tiered_open": 36.628943204920006, "cc_tiered_sealed": 24.915143721783284, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 27.98616472792205 } }, "raw_on_A": { "calibration": 83.94383086837013, "capability": 57.71827121070453, "intelligence": 31.49271155303893 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.014666666666666717, "n": 750 }, "noul": { "invalid_rate": 0.029333333333333322, "n": 375 }, "score": { "invalid_rate": 0.01866666666666672, "n": 375 } }, "views": { "Balanced 33:33:33": 16.62869989264526, "Emphasis on Accuracy 60:20:20": 14.53757347040618, "Emphasis on Cost 20:20:60": 16.08558260894422, "Emphasis on Speed 20:60:20": 20.21989737661098, "Intelligence only": 12.230517698051182 } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Nandakishor M", "axes": { "calibration": 62.93608607777928, "cost": 61.47445744011434, "intelligence": 3.0532964296406795, "speed": 89.45598874418484 }, "calibration": { "parts": { "choice": { "ece_hard": 0.20193798693071482, "ece_score": 59.61240261385703, "mean_tvd": null, "n": 750, "n_ece_hard": 328, "n_tvd": 0, "score": 59.61240261385703 }, "noul": { "abstention_rate": 0.6799999999999999, "brier": 0.28375891613054305, "ece": 0.17069524022939242, "n": 375, "score": 65.86095195412152 }, "score": { "interval90_coverage": 0.728, "interval90_width": 0.4346666666666666, "n": 375, "nrps": 0.195768816097399, "score": 63.33490366535929, "top_ece": 0.2687665552977076 } }, "score": 62.93608607777928 }, "capability": 32.99469125370998, "composite_ci95": { "A": [ 0.0035937243990094452, 0.23534462793689395 ], "B": [ 0.00231035635955886, 0.16355006319549895 ], "C": [ 0.002495641943756559, 0.16343376940062082 ] }, "cost": { "basis": "v1.5-M2 base-model reference: Vega 0.8B is a decision engine on a frozen Qwen3.5-0.8B backbone (Qwen/Qwen3.5-0.8B, resolves in pricing_v15 to deepinfra:Qwen/Qwen3.5-0.8B). DeepInfra list price USD 0.01/M input, USD 0.05/M output from the dated DeepInfra catalog snapshot 2026-09-25 (jevbench-v15-measure-20260925/receipts/DEEPINFRA-MODELS-20260925.json, sha256 84eb7926acd4e93b71d695be24898e2ca999b20011d1b66cc0348ce7c6c3546f; source https://api.deepinfra.com/models/list), read by the frozen pricing_v15 on 2026-10-10. The model generates no text: 0 output tokens, cost = input tokens x USD 0.01/M.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.019239100000000002 }, "display": "Vega 0.8B", "endpoint_condition": "evaluator-owned Lium RTX A6000 48 GB pod (Kansas City, US), offline after setup (egress cut and proven), reviewed vegaml==0.6.1 wheel, backbone bf16 / engine fp32, context 73,728 (longer states cut on the right), reference PyTorch path for linear attention (no flash-linear-attention / causal-conv1d)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 4.066515604051393, "I_sealed": 2.040077255229966, "base": 3.0532964296406795, "excess": -8.046806232518286, "gap": 2.0264383488214266, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 12.19954681215418, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 14.378378378378391, "hard": 9.405473419314262, "judge": 16.94133157547792, "standard": 9.585600669736293 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 14.28571428571428, "hard": -73.91304347826086, "judge": -50.0, "standard": -55.55555555555556 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 46.625135136625836, "hard": -0.915170580118807, "judge": -9.170003719084164, "standard": -19.439472746148546 } }, "sealed|choice": { "cc": 6.1202317656898995, "n": { "easy": 72, "hard": 260, "judge": 141, "standard": 127 }, "tiers": { "easy": -8.874752320367087, "hard": 18.874354182697587, "judge": 3.2012581807112888, "standard": -7.512060647829066 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 39, "hard": 132, "judge": 72, "standard": 57 }, "tiers": { "easy": -23.076923076923073, "hard": -69.6969696969697, "judge": -58.33333333333333, "standard": -78.94736842105263 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 33, "hard": 115, "judge": 75, "standard": 77 }, "tiers": { "easy": -0.09009355832638732, "hard": -3.84788500990465, "judge": -4.989943518704232, "standard": -15.600158434735768 } } } }, "jevbench_score": 0.04022166962784317, "key": "vegaml-0.8b", "last_measured_on": "2026-10-10", "licence": "vegaml package Apache-2.0; HF checkpoint repo declares no licence field (implied, not declared); backbone Qwen3.5 Apache-2.0", "listing": "ranked", "measured_in": "v1.6.5", "measurement_date_status": "measured", "model_pin": "nandakishorm/vega-08b-public-intents@73a76a4f6c4fe672e0daa2d45ddc355afc5bc9dd", "noul_decisive": { "acc_among_decisive": 0.6333333333333333, "decisive": 120, "decisive_rate": 0.32, "n": 375, "supported": true, "valid": 375 }, "planned_weight_denominator": 1.0, "rank": 15, "ranked": true, "ranks": { "A": 15, "B": 15, "C": 15, "capability": 14 }, "repo": "https://huggingface.co/nandakishorm/vega-08b-public-intents", "scores": { "A": 0.04022166962784317, "B": 0.0266984340300977, "C": 0.027931715019335536 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3191188830882311, "p50_s_raw": 0.08455944154411554, "p95_s_adjusted": 0.35518009765073655, "p95_s_raw": 0.10259004882536829 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns": { "family": { "calibration": { "accuracy": 0.27586206896551724, "cc": -19.36556571427138, "cc_by_type": { "choice": 25.0, "noul": -66.66666666666667, "score": -16.43003047614746 }, "low_n": true, "n": 29, "n_by_type": { "choice": 16, "noul": 6, "score": 7 }, "score": 0.0 }, "calibration_gen": { "accuracy": 0.19642857142857142, "cc": -24.988246708550232, "cc_by_type": { "choice": -4.813664596273289, "noul": -76.92307692307692, "score": 6.772001393699512 }, "low_n": false, "n": 112, "n_by_type": { "choice": 56, "noul": 26, "score": 30 }, "score": 0.0 }, "composite_gen": { "accuracy": 0.15306122448979592, "cc": -36.524602578293646, "cc_by_type": { "choice": -3.7037037037037055, "noul": -100.0, "score": -5.870104031177226 }, "low_n": false, "n": 98, "n_by_type": { "choice": 36, "noul": 28, "score": 34 }, "score": 0.0 }, "fairness": { "accuracy": 0.2222222222222222, "cc": -26.181811039531453, "cc_by_type": { "choice": -5.882352941176475, "noul": -100.0, "score": 27.336919822582118 }, "low_n": true, "n": 27, "n_by_type": { "choice": 12, "noul": 6, "score": 9 }, "score": 0.0 }, "jaggedness": { "accuracy": 0.2864864864864865, "cc": -17.791678833140427, "cc_by_type": { "choice": 4.12227883279296, "noul": -23.076923076923073, "score": -34.42039225529116 }, "low_n": false, "n": 185, "n_by_type": { "choice": 97, "noul": 39, "score": 49 }, "score": 0.0 }, "llm_router": { "accuracy": 0.22348484848484848, "cc": -10.709273113952321, "cc_by_type": { "choice": -3.111936832326967, "noul": -47.945205479452056, "score": 18.929322969922058 }, "low_n": false, "n": 264, "n_by_type": { "choice": 131, "noul": 73, "score": 60 }, "score": 0.0 }, "long_state": { "accuracy": 0.1724137931034483, "cc": -34.941907856494, "cc_by_type": { "choice": 4.761904761904764, "noul": -81.81818181818181, "score": -27.769446513204944 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 11, "score": 7 }, "score": 0.0 }, "ml_usecase": { "accuracy": 0.41935483870967744, "cc": -14.749178737264756, "cc_by_type": { "choice": 7.692307692307688, "noul": 25.0, "score": -76.93984390410196 }, "low_n": false, "n": 31, "n_by_type": { "choice": 15, "noul": 8, "score": 8 }, "score": 0.0 }, "multilingual": { "accuracy": 0.2634920634920635, "cc": -18.949056458195674, "cc_by_type": { "choice": 12.225705329153609, "noul": -67.81609195402298, "score": -1.2567827497176554 }, "low_n": false, "n": 315, "n_by_type": { "choice": 146, "noul": 87, "score": 82 }, "score": 0.0 }, "negation_pair_gen": { "accuracy": 0.19642857142857142, "cc": -42.402473689854006, "cc_by_type": { "choice": -9.523809523809524, "noul": -60.0, "score": -57.6836115457525 }, "low_n": false, "n": 56, "n_by_type": { "choice": 28, "noul": 20, "score": 8 }, "score": 0.0 }, "option_order": { "accuracy": 0.3157894736842105, "cc": 13.42952275249723, "cc_by_type": { "choice": 13.42952275249723 }, "low_n": false, "n": 57, "n_by_type": { "choice": 57 }, "score": 13.42952275249723 }, "perturb_pair": { "accuracy": 0.45, "cc": 6.179349639681301, "cc_by_type": { "choice": 29.22673656618611, "noul": -14.28571428571429, "score": 3.5970266385720873 }, "low_n": false, "n": 60, "n_by_type": { "choice": 34, "noul": 14, "score": 12 }, "score": 6.179349639681301 }, "usecase": { "accuracy": 0.29535864978902954, "cc": -19.00704443525642, "cc_by_type": { "choice": 21.37213335902873, "noul": -71.9298245614035, "score": -6.46344210339449 }, "low_n": false, "n": 237, "n_by_type": { "choice": 111, "noul": 57, "score": 69 }, "score": 0.0 } }, "lang": { "ar": { "accuracy": 0.45454545454545453, "cc": -7.714580910237068, "cc_by_type": { "choice": -6.666666666666665, "noul": 0.0, "score": -16.47707606404454 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 4, "score": 2 }, "score": 0.0 }, "cs": { "accuracy": 0.25, "cc": -38.135593220338976, "cc_by_type": { "choice": 23.72881355932204, "noul": -100.0 }, "low_n": true, "n": 8, "n_by_type": { "choice": 5, "noul": 3 }, "score": 0.0 }, "da": { "accuracy": 0.2916666666666667, "cc": -23.813705445205983, "cc_by_type": { "choice": 5.325443786982243, "noul": -100.0, "score": 23.23343987739981 }, "low_n": true, "n": 24, "n_by_type": { "choice": 11, "noul": 5, "score": 8 }, "score": 0.0 }, "de": { "accuracy": 0.26666666666666666, "cc": -9.48435284148812, "cc_by_type": { "choice": 13.095956547978272, "noul": -33.333333333333336, "score": -8.215681739109293 }, "low_n": false, "n": 45, "n_by_type": { "choice": 24, "noul": 9, "score": 12 }, "score": 0.0 }, "el": { "accuracy": 0.4444444444444444, "cc": -38.08299402731068, "cc_by_type": { "choice": 39.7590361445783, "noul": -60.0, "score": -94.00801822651034 }, "low_n": true, "n": 18, "n_by_type": { "choice": 11, "noul": 5, "score": 2 }, "score": 0.0 }, "en": { "accuracy": 0.2529043789097408, "cc": -19.4845221445307, "cc_by_type": { "choice": 6.449803701626394, "noul": -60.902255639097746, "score": -4.00111449612075 }, "low_n": false, "n": 1119, "n_by_type": { "choice": 577, "noul": 266, "score": 276 }, "score": 0.0 }, "es": { "accuracy": 0.35714285714285715, "cc": -3.020542459003753, "cc_by_type": { "choice": -17.81818181818182, "noul": 9.090909090909083, "score": -0.3343546497385219 }, "low_n": true, "n": 28, "n_by_type": { "choice": 10, "noul": 11, "score": 7 }, "score": 0.0 }, "fi": { "accuracy": 0.4444444444444444, "cc": 1.6431924882629083, "cc_by_type": { "choice": 36.61971830985915, "noul": -33.333333333333336 }, "low_n": true, "n": 9, "n_by_type": { "choice": 6, "noul": 3 }, "score": 1.6431924882629083 }, "fr": { "accuracy": 0.06666666666666667, "cc": -42.623688574546726, "cc_by_type": { "choice": -26.93935119887165, "noul": -100.0, "score": -0.9317145247685232 }, "low_n": true, "n": 15, "n_by_type": { "choice": 5, "noul": 4, "score": 6 }, "score": 0.0 }, "hi": { "accuracy": 0.2, "cc": -28.83602734828803, "cc_by_type": { "choice": -11.801242236024846, "noul": -40.0, "score": -34.70683980883924 }, "low_n": false, "n": 30, "n_by_type": { "choice": 11, "noul": 10, "score": 9 }, "score": 0.0 }, "id": { "accuracy": 0.0, "cc": -64.51612903225806, "cc_by_type": { "choice": -29.032258064516125, "noul": -100.0 }, "low_n": true, "n": 6, "n_by_type": { "choice": 4, "noul": 2 }, "score": 0.0 }, "it": { "accuracy": 0.5294117647058824, "cc": 5.113250586139728, "cc_by_type": { "choice": 49.22425952045134, "noul": 0.0, "score": -33.88450776203216 }, "low_n": true, "n": 17, "n_by_type": { "choice": 5, "noul": 4, "score": 8 }, "score": 5.113250586139728 }, "ja": { "accuracy": 0.21875, "cc": -21.637722673747195, "cc_by_type": { "choice": 2.4390243902439046, "noul": -71.42857142857143, "score": 4.076379017085941 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 0.0 }, "ko": { "accuracy": 0.18181818181818182, "cc": -20.186550397257573, "cc_by_type": { "choice": 1.6393442622950836, "noul": 0.0, "score": -62.1989954540678 }, "low_n": true, "n": 11, "n_by_type": { "choice": 4, "noul": 2, "score": 5 }, "score": 0.0 }, "mixed": { "accuracy": 0.2727272727272727, "cc": -1.2255967997920887, "cc_by_type": { "choice": -9.890109890109892, "noul": -50.0, "score": 56.213319490733625 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 4, "score": 1 }, "score": 0.0 }, "nl": { "accuracy": 0.07692307692307693, "cc": -33.67879710336313, "cc_by_type": { "choice": -0.2785515320334245, "noul": -100.0, "score": -0.7578397780559687 }, "low_n": true, "n": 13, "n_by_type": { "choice": 5, "noul": 6, "score": 2 }, "score": 0.0 }, "no": { "accuracy": 0.0, "cc": -65.57377049180329, "cc_by_type": { "choice": -31.14754098360656, "noul": -100.0 }, "low_n": true, "n": 7, "n_by_type": { "choice": 4, "noul": 3 }, "score": 0.0 }, "pl": { "accuracy": 0.275, "cc": -16.97531167960543, "cc_by_type": { "choice": 29.164844774814167, "noul": -80.0, "score": -0.0907798136304594 }, "low_n": false, "n": 40, "n_by_type": { "choice": 16, "noul": 10, "score": 14 }, "score": 0.0 }, "pt": { "accuracy": 0.5, "cc": 21.729761287554258, "cc_by_type": { "choice": 81.90954773869345, "noul": -66.66666666666667, "score": 49.946402790636014 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 6, "score": 5 }, "score": 21.729761287554258 }, "sv": { "accuracy": 0.25, "cc": -8.670532037806744, "cc_by_type": { "choice": 11.111111111111109, "noul": -100.0, "score": 62.87729277546865 }, "low_n": true, "n": 8, "n_by_type": { "choice": 3, "noul": 4, "score": 1 }, "score": 0.0 }, "tr": { "accuracy": 0.18181818181818182, "cc": -58.40033972849334, "cc_by_type": { "choice": -5.26315789473684, "noul": -100.0, "score": -69.93786129074316 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 3, "score": 3 }, "score": 0.0 }, "uk": { "accuracy": 0.18181818181818182, "cc": -43.03199883446496, "cc_by_type": { "choice": 13.043478260869565, "noul": -100.0, "score": -42.13947476426443 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 1, "score": 4 }, "score": 0.0 }, "zh": { "accuracy": 0.5, "cc": 18.615495552140683, "cc_by_type": { "choice": 35.13513513513514, "noul": -33.333333333333336, "score": 54.04468485462024 }, "low_n": true, "n": 8, "n_by_type": { "choice": 4, "noul": 3, "score": 1 }, "score": 18.615495552140683 } }, "type": { "choice": { "accuracy": 0.2866666666666667, "cc": 7.583347728450512, "cc_by_type": { "choice": 7.583347728450512 }, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 7.583347728450512 }, "noul": { "accuracy": 0.20266666666666666, "cc": -59.46666666666667, "cc_by_type": { "noul": -59.46666666666667 }, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.264, "cc": -4.914138326410811, "cc_by_type": { "score": -4.914138326410811 }, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "usecase": { "advertising": { "accuracy": 0.25, "cc": -42.23115369530359, "cc_by_type": { "choice": 11.037891268533771, "noul": -100.0, "score": -37.73135235444454 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 0.0 }, "code_linting": { "accuracy": 0.4166666666666667, "cc": 10.053783668961861, "cc_by_type": { "choice": 21.440261865793783, "noul": -19.999999999999996, "score": 28.721089141091795 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 10.053783668961861 }, "customer_support": { "accuracy": 0.25, "cc": -14.695878083746338, "cc_by_type": { "choice": -12.90322580645161, "noul": -50.0, "score": 18.815591555212595 }, "low_n": true, "n": 20, "n_by_type": { "choice": 8, "noul": 8, "score": 4 }, "score": 0.0 }, "demand_forecasting": { "accuracy": 0.07692307692307693, "cc": -52.52931897614329, "cc_by_type": { "choice": -20.481927710843376, "noul": -100.0, "score": -37.10602921758648 }, "low_n": true, "n": 26, "n_by_type": { "choice": 11, "noul": 4, "score": 11 }, "score": 0.0 }, "ecommerce_marketplaces": { "accuracy": 0.1111111111111111, "cc": -38.90478330755122, "cc_by_type": { "choice": 6.832298136645961, "noul": -100.0, "score": -23.546648059299624 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 5, "score": 6 }, "score": 0.0 }, "feature_extraction": { "accuracy": 0.3793103448275862, "cc": -13.328851245398967, "cc_by_type": { "choice": 31.623931623931618, "noul": -60.0, "score": -11.61048536012852 }, "low_n": true, "n": 29, "n_by_type": { "choice": 15, "noul": 5, "score": 9 }, "score": 0.0 }, "financial_crime": { "accuracy": 0.34615384615384615, "cc": -18.503379574597428, "cc_by_type": { "choice": 47.82608695652174, "noul": -75.0, "score": -28.33622568031402 }, "low_n": true, "n": 26, "n_by_type": { "choice": 10, "noul": 8, "score": 8 }, "score": 0.0 }, "gaming": { "accuracy": 0.27586206896551724, "cc": -25.77506638432793, "cc_by_type": { "choice": -6.9306930693069315, "noul": -71.42857142857143, "score": 1.0340653448945747 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 7, "score": 11 }, "score": 0.0 }, "insurance_claims": { "accuracy": 0.14285714285714285, "cc": -30.119048798099303, "cc_by_type": { "choice": -11.11111111111111, "noul": -100.0, "score": 20.7539647168132 }, "low_n": true, "n": 14, "n_by_type": { "choice": 8, "noul": 5, "score": 1 }, "score": 0.0 }, "knowledge_graphs": { "accuracy": 0.17391304347826086, "cc": -34.69531570651282, "cc_by_type": { "choice": -0.7633587786259586, "noul": -100.0, "score": -3.3225883409125245 }, "low_n": true, "n": 23, "n_by_type": { "choice": 14, "noul": 5, "score": 4 }, "score": 0.0 }, "lead_generation": { "accuracy": 0.48, "cc": 8.261797778580346, "cc_by_type": { "choice": 67.47967479674797, "noul": -25.0, "score": -17.694281461006934 }, "low_n": true, "n": 25, "n_by_type": { "choice": 8, "noul": 8, "score": 9 }, "score": 8.261797778580346 }, "legal_compliance": { "accuracy": 0.23076923076923078, "cc": -31.134132674360444, "cc_by_type": { "choice": 18.032786885245905, "noul": -100.0, "score": -11.435184908327244 }, "low_n": true, "n": 13, "n_by_type": { "choice": 8, "noul": 1, "score": 4 }, "score": 0.0 }, "llm_guardrails": { "accuracy": 0.3103448275862069, "cc": -9.67272641203819, "cc_by_type": { "choice": 17.431192660550458, "noul": -75.0, "score": 28.550628103334972 }, "low_n": true, "n": 29, "n_by_type": { "choice": 14, "noul": 8, "score": 7 }, "score": 0.0 }, "model_routing": { "accuracy": 0.29333333333333333, "cc": 2.0858721093715658, "cc_by_type": { "choice": 1.9029750737067974, "noul": -18.181818181818176, "score": 22.536459436226075 }, "low_n": false, "n": 150, "n_by_type": { "choice": 74, "noul": 44, "score": 32 }, "score": 2.0858721093715658 }, "moderation_trust_safety": { "accuracy": 0.3125, "cc": -22.66548316017918, "cc_by_type": { "choice": 36.84210526315789, "noul": -100.0, "score": -4.8385547436954335 }, "low_n": true, "n": 16, "n_by_type": { "choice": 8, "noul": 5, "score": 3 }, "score": 0.0 }, "recruiting": { "accuracy": 0.38095238095238093, "cc": 1.0762044705122094, "cc_by_type": { "choice": 69.84924623115576, "noul": -66.66666666666667, "score": 0.046033847047532994 }, "low_n": true, "n": 21, "n_by_type": { "choice": 9, "noul": 6, "score": 6 }, "score": 1.0762044705122094 }, "risk_assessment": { "accuracy": 0.3076923076923077, "cc": -21.111851703106247, "cc_by_type": { "choice": 24.528301886792452, "noul": -100.0, "score": 12.136143003888812 }, "low_n": true, "n": 13, "n_by_type": { "choice": 7, "noul": 3, "score": 3 }, "score": 0.0 }, "scientific_discovery": { "accuracy": 0.1875, "cc": -22.40823864613203, "cc_by_type": { "choice": 4.382470119521911, "noul": -71.42857142857143, "score": -0.17861462934656291 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 0.0 }, "search_retrieval": { "accuracy": 0.2727272727272727, "cc": -35.0856709518421, "cc_by_type": { "choice": 14.08114558472553, "noul": -66.66666666666667, "score": -52.671491773585146 }, "low_n": true, "n": 22, "n_by_type": { "choice": 9, "noul": 6, "score": 7 }, "score": 0.0 } } }, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 57.40975035799149, 66.49290908060016 ], "I": [ 1.3348314800373893, 5.689355834192679 ], "capability": [ 29.933677735120224, 35.37515714416046 ], "composite": { "A": [ 0.0035937243990094452, 0.23534462793689395 ], "B": [ 0.00231035635955886, 0.16355006319549895 ], "C": [ 0.002495641943756559, 0.16343376940062082 ] }, "seed": 29 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": false, "g_med_used": 10.073244581339713, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 62.93608607777928, "capability": 32.99469125370998, "intelligence": 3.0532964296406795 }, "per_type": { "choice": { "accuracy": 0.2866666666666667, "calibration": 59.61240261385703, "cc": 7.583347728450512, "cc_by_type": { "choice": 7.583347728450512 }, "cc_tiered": 9.15988928892204, "cc_tiered_open": 12.19954681215418, "cc_tiered_sealed": 6.1202317656898995, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 7.583347728450512 }, "noul": { "accuracy": 0.20266666666666666, "calibration": 65.86095195412152, "cc": -59.46666666666667, "cc_by_type": { "noul": -59.46666666666667 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.264, "calibration": 63.33490366535929, "cc": -4.914138326410811, "cc_by_type": { "score": -4.914138326410811 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 59.68341710343884, "capability": 30.94517056072779, "intelligence": 2.206924018016739 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.03151658754618596, "Emphasis on Accuracy 60:20:20": 0.018460810494659353, "Emphasis on Cost 20:20:60": 0.048117449859240324, "Emphasis on Speed 20:60:20": 0.04941520456803552, "Intelligence only": 0.011385887829608154 } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Nandakishor M", "axes": { "calibration": 58.68489087403218, "cost": 47.16081979852447, "intelligence": 16.689214224692094, "speed": 84.39386641436894 }, "calibration": { "parts": { "choice": { "ece_hard": 0.2574125103434302, "ece_score": 48.517497931313954, "mean_tvd": null, "n": 750, "n_ece_hard": 328, "n_tvd": 0, "score": 48.517497931313954 }, "noul": { "abstention_rate": 0.30400000000000005, "brier": 0.2781281471262366, "ece": 0.2285300562531464, "n": 375, "score": 54.293988749370726 }, "score": { "interval90_coverage": 0.8346666666666667, "interval90_width": 0.49288888888888904, "n": 375, "nrps": 0.16226214991492005, "score": 73.24318594141187, "top_ece": 0.18643706562842138 } }, "score": 58.68489087403218 }, "capability": 37.687052549362136, "composite_ci95": { "A": [ 1.5678216817255275, 6.594029572003121 ], "B": [ 1.2095817426628035, 5.54602239255017 ], "C": [ 1.0887650567538385, 4.579187202779947 ] }, "cost": { "basis": "v1.5-M2 base-model reference: Vega 4B is a decision engine on a frozen Qwen3.5-4B backbone (Qwen/Qwen3.5-4B, resolves in pricing_v15 to deepinfra:Qwen/Qwen3.5-4B). DeepInfra list price USD 0.03/M input, USD 0.15/M output from the dated DeepInfra catalog snapshot 2026-09-25 (jevbench-v15-measure-20260925/receipts/DEEPINFRA-MODELS-20260925.json, sha256 84eb7926acd4e93b71d695be24898e2ca999b20011d1b66cc0348ce7c6c3546f; source https://api.deepinfra.com/models/list), read by the frozen pricing_v15 on 2026-10-10. The listing is flagged by pricing_v15 as deprecated after the 25 Sep cut-off (DEPRECATED_AFTER_CUTOFF) and stays frozen with that note. The model generates no text: 0 output tokens, cost = input tokens x USD 0.03/M.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.057717300000000006 }, "display": "Vega 4B", "endpoint_condition": "evaluator-owned Lium RTX A6000 48 GB pod (Des Moines, US), offline after setup (egress cut and proven), reviewed vegaml==0.6.1 wheel, backbone bf16 / engine fp32, context 73,728 (longer states cut on the right), reference PyTorch path for linear attention (no flash-linear-attention / causal-conv1d)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 22.023529046873755, "I_sealed": 11.354899402510437, "base": 16.689214224692094, "excess": 0.5953850630236044, "gap": 10.668629644363318, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 42.15141777710706, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 45.51351351351353, "hard": 30.16671909405474, "judge": 48.58272907053395, "standard": 54.79280033486815 } }, "open|noul": { "cc": 9.665976535541756, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": 30.434782608695656, "judge": -19.999999999999996, "standard": -11.111111111111116 } }, "open|score": { "cc": 14.253192827972441, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 36.64297240164319, "hard": 6.501670939484594, "judge": 20.791162375101624, "standard": 8.754392497418984 } }, "sealed|choice": { "cc": 25.42491346722165, "n": { "easy": 72, "hard": 260, "judge": 141, "standard": 127 }, "tiers": { "easy": 32.4225675252894, "hard": 23.821039903264804, "judge": 22.378367409060942, "standard": 29.70365265334253 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 39, "hard": 132, "judge": 72, "standard": 57 }, "tiers": { "easy": 7.692307692307687, "hard": -22.72727272727273, "judge": 8.333333333333325, "standard": -12.28070175438597 } }, "sealed|score": { "cc": 8.639784740309665, "n": { "easy": 33, "hard": 115, "judge": 75, "standard": 77 }, "tiers": { "easy": 23.787017462891814, "hard": 6.903920787097362, "judge": 10.43509835324301, "standard": 1.8449258660431789 } } } }, "jevbench_score": 3.603902446043355, "key": "vegaml-4b", "last_measured_on": "2026-10-10", "licence": "vegaml package Apache-2.0; HF checkpoint repo declares no licence field (implied, not declared); backbone Qwen3.5 Apache-2.0", "listing": "ranked", "measured_in": "v1.6.5", "measurement_date_status": "measured", "model_pin": "nandakishorm/vega-08b-public-intents@73a76a4f6c4fe672e0daa2d45ddc355afc5bc9dd", "noul_decisive": { "acc_among_decisive": 0.6819923371647509, "decisive": 261, "decisive_rate": 0.696, "n": 375, "supported": true, "valid": 375 }, "planned_weight_denominator": 1.0, "rank": 11, "ranked": true, "ranks": { "A": 11, "B": 11, "C": 11, "capability": 12 }, "repo": "https://huggingface.co/nandakishorm/vega-08b-public-intents", "scores": { "A": 3.603902446043355, "B": 2.9164270830443, "C": 2.502710031974551 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.5304726889356971, "p50_s_raw": 0.19023634446784854, "p95_s_adjusted": 0.6854098306968808, "p95_s_raw": 0.2677049153484404 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns": { "family": { "calibration": { "accuracy": 0.5862068965517241, "cc": 32.10254092129096, "cc_by_type": { "choice": 50.0, "noul": 0.0, "score": 46.30762276387288 }, "low_n": true, "n": 29, "n_by_type": { "choice": 16, "noul": 6, "score": 7 }, "score": 32.10254092129096 }, "calibration_gen": { "accuracy": 0.3482142857142857, "cc": -7.764599070714067, "cc_by_type": { "choice": 16.14906832298137, "noul": -38.46153846153846, "score": -0.9813270735851143 }, "low_n": false, "n": 112, "n_by_type": { "choice": 56, "noul": 26, "score": 30 }, "score": 0.0 }, "composite_gen": { "accuracy": 0.35714285714285715, "cc": 4.1474482072018475, "cc_by_type": { "choice": 7.407407407407411, "noul": -7.14285714285714, "score": 12.177794357055271 }, "low_n": false, "n": 98, "n_by_type": { "choice": 36, "noul": 28, "score": 34 }, "score": 4.1474482072018475 }, "fairness": { "accuracy": 0.7037037037037037, "cc": 46.819844941220424, "cc_by_type": { "choice": 64.70588235294117, "noul": 0.0, "score": 75.75365247072008 }, "low_n": true, "n": 27, "n_by_type": { "choice": 12, "noul": 6, "score": 9 }, "score": 46.819844941220424 }, "jaggedness": { "accuracy": 0.34054054054054056, "cc": -9.581651647472881, "cc_by_type": { "choice": 19.407132931912926, "noul": -28.205128205128204, "score": -19.946959669203366 }, "low_n": false, "n": 185, "n_by_type": { "choice": 97, "noul": 39, "score": 49 }, "score": 0.0 }, "llm_router": { "accuracy": 0.5303030303030303, "cc": 35.11456970853259, "cc_by_type": { "choice": 34.9744542498839, "noul": 28.767123287671236, "score": 41.602131588042646 }, "low_n": false, "n": 264, "n_by_type": { "choice": 131, "noul": 73, "score": 60 }, "score": 35.11456970853259 }, "long_state": { "accuracy": 0.3448275862068966, "cc": -4.35719562611893, "cc_by_type": { "choice": 16.66666666666667, "noul": -27.27272727272727, "score": -2.4655262722961924 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 11, "score": 7 }, "score": 0.0 }, "ml_usecase": { "accuracy": 0.3548387096774194, "cc": -22.26927167429365, "cc_by_type": { "choice": 35.38461538461538, "noul": -75.0, "score": -27.192430407496325 }, "low_n": false, "n": 31, "n_by_type": { "choice": 15, "noul": 8, "score": 8 }, "score": 0.0 }, "multilingual": { "accuracy": 0.4603174603174603, "cc": 15.9575022436564, "cc_by_type": { "choice": 35.512763098970005, "noul": -1.1494252873563204, "score": 13.509168919355519 }, "low_n": false, "n": 315, "n_by_type": { "choice": 146, "noul": 87, "score": 82 }, "score": 15.9575022436564 }, "negation_pair_gen": { "accuracy": 0.32142857142857145, "cc": -18.937881655648187, "cc_by_type": { "choice": -9.523809523809524, "noul": 0.0, "score": -47.28983544313503 }, "low_n": false, "n": 56, "n_by_type": { "choice": 28, "noul": 20, "score": 8 }, "score": 0.0 }, "option_order": { "accuracy": 0.5614035087719298, "cc": 44.50610432852386, "cc_by_type": { "choice": 44.50610432852386 }, "low_n": false, "n": 57, "n_by_type": { "choice": 57 }, "score": 44.50610432852386 }, "perturb_pair": { "accuracy": 0.5333333333333333, "cc": 41.1419841739421, "cc_by_type": { "choice": 25.29488859764089, "noul": 42.85714285714286, "score": 55.273921067042544 }, "low_n": false, "n": 60, "n_by_type": { "choice": 34, "noul": 14, "score": 12 }, "score": 41.1419841739421 }, "usecase": { "accuracy": 0.379746835443038, "cc": -1.691456792175881, "cc_by_type": { "choice": 28.30988629793796, "noul": -22.807017543859654, "score": -10.577239130605953 }, "low_n": false, "n": 237, "n_by_type": { "choice": 111, "noul": 57, "score": 69 }, "score": 0.0 } }, "lang": { "ar": { "accuracy": 0.5454545454545454, "cc": -14.905334686220934, "cc_by_type": { "choice": 73.33333333333334, "noul": -50.0, "score": -68.04933739199615 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 4, "score": 2 }, "score": 0.0 }, "cs": { "accuracy": 0.375, "cc": -4.802259887005647, "cc_by_type": { "choice": 23.72881355932204, "noul": -33.333333333333336 }, "low_n": true, "n": 8, "n_by_type": { "choice": 5, "noul": 3 }, "score": 0.0 }, "da": { "accuracy": 0.4583333333333333, "cc": 22.143800617963134, "cc_by_type": { "choice": 64.49704142011835, "noul": 19.999999999999996, "score": -18.065639566228953 }, "low_n": true, "n": 24, "n_by_type": { "choice": 11, "noul": 5, "score": 8 }, "score": 22.143800617963134 }, "de": { "accuracy": 0.4444444444444444, "cc": 3.742298256027752, "cc_by_type": { "choice": 29.390464695232346, "noul": -33.333333333333336, "score": 15.169763406184245 }, "low_n": false, "n": 45, "n_by_type": { "choice": 24, "noul": 9, "score": 12 }, "score": 3.742298256027752 }, "el": { "accuracy": 0.5, "cc": -7.267096986083452, "cc_by_type": { "choice": 51.80722891566265, "noul": -60.0, "score": -13.608519873913005 }, "low_n": true, "n": 18, "n_by_type": { "choice": 11, "noul": 5, "score": 2 }, "score": 0.0 }, "en": { "accuracy": 0.4289544235924933, "cc": 10.591540002599032, "cc_by_type": { "choice": 26.86483454851368, "noul": -5.263157894736848, "score": 10.17294335402027 }, "low_n": false, "n": 1119, "n_by_type": { "choice": 577, "noul": 266, "score": 276 }, "score": 10.591540002599032 }, "es": { "accuracy": 0.5357142857142857, "cc": 23.735811235182705, "cc_by_type": { "choice": 47.63636363636363, "noul": 9.090909090909083, "score": 14.480160978275391 }, "low_n": true, "n": 28, "n_by_type": { "choice": 10, "noul": 11, "score": 7 }, "score": 23.735811235182705 }, "fi": { "accuracy": 0.2222222222222222, "cc": -19.483568075117372, "cc_by_type": { "choice": -5.63380281690141, "noul": -33.333333333333336 }, "low_n": true, "n": 9, "n_by_type": { "choice": 6, "noul": 3 }, "score": 0.0 }, "fr": { "accuracy": 0.5333333333333333, "cc": 9.81849210967231, "cc_by_type": { "choice": 49.22425952045134, "noul": -50.0, "score": 30.231216808565588 }, "low_n": true, "n": 15, "n_by_type": { "choice": 5, "noul": 4, "score": 6 }, "score": 9.81849210967231 }, "hi": { "accuracy": 0.36666666666666664, "cc": -2.9723800154374516, "cc_by_type": { "choice": 37.888198757763966, "noul": -40.0, "score": -6.805338804076322 }, "low_n": false, "n": 30, "n_by_type": { "choice": 11, "noul": 10, "score": 9 }, "score": 0.0 }, "id": { "accuracy": 0.16666666666666666, "cc": -14.516129032258062, "cc_by_type": { "choice": -29.032258064516125, "noul": 0.0 }, "low_n": true, "n": 6, "n_by_type": { "choice": 4, "noul": 2 }, "score": 0.0 }, "it": { "accuracy": 0.5882352941176471, "cc": 41.890588956237984, "cc_by_type": { "choice": 74.61212976022568, "noul": 50.0, "score": 1.059637108488276 }, "low_n": true, "n": 17, "n_by_type": { "choice": 5, "noul": 4, "score": 8 }, "score": 41.890588956237984 }, "ja": { "accuracy": 0.34375, "cc": 6.462836027337623, "cc_by_type": { "choice": -5.691056910569103, "noul": -14.28571428571429, "score": 39.36527927829626 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 6.462836027337623 }, "ko": { "accuracy": 0.36363636363636365, "cc": 18.2762217539951, "cc_by_type": { "choice": 34.42622950819673, "noul": 0.0, "score": 20.402435753788563 }, "low_n": true, "n": 11, "n_by_type": { "choice": 4, "noul": 2, "score": 5 }, "score": 18.2762217539951 }, "mixed": { "accuracy": 0.7272727272727273, "cc": 59.83071388979721, "cc_by_type": { "choice": 56.04395604395604, "noul": 100.0, "score": 23.448185625435602 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 4, "score": 1 }, "score": 59.83071388979721 }, "nl": { "accuracy": 0.15384615384615385, "cc": -18.26355956150215, "cc_by_type": { "choice": -25.34818941504178, "noul": -66.66666666666667, "score": 37.224177397202 }, "low_n": true, "n": 13, "n_by_type": { "choice": 5, "noul": 6, "score": 2 }, "score": 0.0 }, "no": { "accuracy": 0.7142857142857143, "cc": 67.21311475409837, "cc_by_type": { "choice": 34.42622950819673, "noul": 100.0 }, "low_n": true, "n": 7, "n_by_type": { "choice": 4, "noul": 3 }, "score": 67.21311475409837 }, "pl": { "accuracy": 0.525, "cc": 43.18418848568871, "cc_by_type": { "choice": 68.51770878880629, "noul": 60.00000000000001, "score": 1.0348566682598204 }, "low_n": false, "n": 40, "n_by_type": { "choice": 16, "noul": 10, "score": 14 }, "score": 43.18418848568871 }, "pt": { "accuracy": 0.4444444444444444, "cc": 15.083741051175762, "cc_by_type": { "choice": 27.638190954773865, "noul": -33.333333333333336, "score": 50.94636553208676 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 6, "score": 5 }, "score": 15.083741051175762 }, "sv": { "accuracy": 0.125, "cc": -24.998495011569247, "cc_by_type": { "choice": -33.333333333333336, "noul": -50.0, "score": 8.337848298625605 }, "low_n": true, "n": 8, "n_by_type": { "choice": 3, "noul": 4, "score": 1 }, "score": 0.0 }, "tr": { "accuracy": 0.6363636363636364, "cc": 20.74230849140807, "cc_by_type": { "choice": 47.368421052631575, "noul": 100.0, "score": -85.14149557840736 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 3, "score": 3 }, "score": 20.74230849140807 }, "uk": { "accuracy": 0.18181818181818182, "cc": -41.58877432033782, "cc_by_type": { "choice": 13.043478260869565, "noul": -100.0, "score": -37.80980122188302 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 1, "score": 4 }, "score": 0.0 }, "zh": { "accuracy": 0.75, "cc": 65.40226528868891, "cc_by_type": { "choice": 67.56756756756756, "noul": 33.33333333333333, "score": 95.30589496516582 }, "low_n": true, "n": 8, "n_by_type": { "choice": 4, "noul": 3, "score": 1 }, "score": 65.40226528868891 } }, "type": { "choice": { "accuracy": 0.4493333333333333, "cc": 28.65779927448609, "cc_by_type": { "choice": 28.65779927448609 }, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 28.65779927448609 }, "noul": { "accuracy": 0.4746666666666667, "cc": -5.066666666666664, "cc_by_type": { "noul": -5.066666666666664 }, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.3626666666666667, "cc": 9.783689886391333, "cc_by_type": { "score": 9.783689886391333 }, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 9.783689886391333 } }, "usecase": { "advertising": { "accuracy": 0.3333333333333333, "cc": -35.1534607494542, "cc_by_type": { "choice": 20.922570016474467, "noul": -100.0, "score": -26.382952264837066 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 0.0 }, "code_linting": { "accuracy": 0.5416666666666666, "cc": 15.916918386842573, "cc_by_type": { "choice": 41.08019639934533, "noul": 19.999999999999996, "score": -13.329441238817608 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 15.916918386842573 }, "customer_support": { "accuracy": 0.45, "cc": 8.926626674691754, "cc_by_type": { "choice": 35.483870967741936, "noul": 25.0, "score": -33.703990943666675 }, "low_n": true, "n": 20, "n_by_type": { "choice": 8, "noul": 8, "score": 4 }, "score": 8.926626674691754 }, "demand_forecasting": { "accuracy": 0.23076923076923078, "cc": -37.51418716919469, "cc_by_type": { "choice": 3.614457831325295, "noul": -100.0, "score": -16.15701933890936 }, "low_n": true, "n": 26, "n_by_type": { "choice": 11, "noul": 4, "score": 11 }, "score": 0.0 }, "ecommerce_marketplaces": { "accuracy": 0.2222222222222222, "cc": -13.643840746791454, "cc_by_type": { "choice": 25.465838509316768, "noul": -60.0, "score": -6.397360749691128 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 5, "score": 6 }, "score": 0.0 }, "feature_extraction": { "accuracy": 0.27586206896551724, "cc": -6.646648832243309, "cc_by_type": { "choice": 5.9829059829059785, "noul": -19.999999999999996, "score": -5.922852479635909 }, "low_n": true, "n": 29, "n_by_type": { "choice": 15, "noul": 5, "score": 9 }, "score": 0.0 }, "financial_crime": { "accuracy": 0.6538461538461539, "cc": 30.68478198913924, "cc_by_type": { "choice": 86.95652173913045, "noul": 25.0, "score": -19.902175771712738 }, "low_n": true, "n": 26, "n_by_type": { "choice": 10, "noul": 8, "score": 8 }, "score": 30.68478198913924 }, "gaming": { "accuracy": 0.27586206896551724, "cc": -8.539638885229605, "cc_by_type": { "choice": -6.9306930693069315, "noul": -42.85714285714286, "score": 24.16891927076098 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 7, "score": 11 }, "score": 0.0 }, "insurance_claims": { "accuracy": 0.21428571428571427, "cc": -26.327496496464544, "cc_by_type": { "choice": 4.761904761904763, "noul": -60.0, "score": -23.744394251298395 }, "low_n": true, "n": 14, "n_by_type": { "choice": 8, "noul": 5, "score": 1 }, "score": 0.0 }, "knowledge_graphs": { "accuracy": 0.21739130434782608, "cc": -7.707530000989269, "cc_by_type": { "choice": -0.7633587786259586, "noul": -19.999999999999996, "score": -2.359231224341851 }, "low_n": true, "n": 23, "n_by_type": { "choice": 14, "noul": 5, "score": 4 }, "score": 0.0 }, "lead_generation": { "accuracy": 0.52, "cc": 19.542316672389997, "cc_by_type": { "choice": 51.21951219512196, "noul": 25.0, "score": -17.592562177951955 }, "low_n": true, "n": 25, "n_by_type": { "choice": 8, "noul": 8, "score": 9 }, "score": 19.542316672389997 }, "legal_compliance": { "accuracy": 0.23076923076923078, "cc": -17.21253914360014, "cc_by_type": { "choice": 18.032786885245905, "noul": -100.0, "score": 30.32959568395368 }, "low_n": true, "n": 13, "n_by_type": { "choice": 8, "noul": 1, "score": 4 }, "score": 0.0 }, "llm_guardrails": { "accuracy": 0.41379310344827586, "cc": -0.8829038829667036, "cc_by_type": { "choice": 35.77981651376147, "noul": -50.0, "score": 11.571471837338422 }, "low_n": true, "n": 29, "n_by_type": { "choice": 14, "noul": 8, "score": 7 }, "score": 0.0 }, "model_routing": { "accuracy": 0.5, "cc": 24.299647249468666, "cc_by_type": { "choice": 42.106673813990895, "noul": 22.72727272727273, "score": 8.064995207142367 }, "low_n": false, "n": 150, "n_by_type": { "choice": 74, "noul": 44, "score": 32 }, "score": 24.299647249468666 }, "moderation_trust_safety": { "accuracy": 0.5, "cc": 21.15669006850672, "cc_by_type": { "choice": 21.052631578947366, "noul": 19.999999999999996, "score": 22.417438626572796 }, "low_n": true, "n": 16, "n_by_type": { "choice": 8, "noul": 5, "score": 3 }, "score": 21.15669006850672 }, "recruiting": { "accuracy": 0.42857142857142855, "cc": 4.28970309222692, "cc_by_type": { "choice": 69.84924623115576, "noul": -33.333333333333336, "score": -23.646803621141665 }, "low_n": true, "n": 21, "n_by_type": { "choice": 9, "noul": 6, "score": 6 }, "score": 4.28970309222692 }, "risk_assessment": { "accuracy": 0.5384615384615384, "cc": 13.524055414252595, "cc_by_type": { "choice": 43.39622641509434, "noul": 33.33333333333333, "score": -36.15739350566989 }, "low_n": true, "n": 13, "n_by_type": { "choice": 7, "noul": 3, "score": 3 }, "score": 13.524055414252595 }, "scientific_discovery": { "accuracy": 0.21875, "cc": -20.974690029781613, "cc_by_type": { "choice": 20.31872509960159, "noul": -71.42857142857143, "score": -11.814223760374997 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 0.0 }, "search_retrieval": { "accuracy": 0.5, "cc": 2.15502085448014, "cc_by_type": { "choice": 57.04057279236276, "noul": -33.333333333333336, "score": -17.242176895589 }, "low_n": true, "n": 22, "n_by_type": { "choice": 9, "noul": 6, "score": 7 }, "score": 2.15502085448014 } } }, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 53.36316060753802, 62.70466652862149 ], "I": [ 12.10983229856278, 21.198741423993066 ], "capability": [ 33.69125607691782, 41.06968699886858 ], "composite": { "A": [ 1.5678216817255275, 6.594029572003121 ], "B": [ 1.2095817426628035, 5.54602239255017 ], "C": [ 1.0887650567538385, 4.579187202779947 ] }, "seed": 30 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": false, "g_med_used": 10.073244581339713, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 58.68489087403218, "capability": 37.687052549362136, "intelligence": 16.689214224692094 }, "per_type": { "choice": { "accuracy": 0.4493333333333333, "calibration": 48.517497931313954, "cc": 28.65779927448609, "cc_by_type": { "choice": 28.65779927448609 }, "cc_tiered": 33.78816562216436, "cc_tiered_open": 42.15141777710706, "cc_tiered_sealed": 25.42491346722165, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 28.65779927448609 }, "noul": { "accuracy": 0.4746666666666667, "calibration": 54.293988749370726, "cc": -5.066666666666664, "cc_by_type": { "noul": -5.066666666666664 }, "cc_tiered": 4.832988267770878, "cc_tiered_open": 9.665976535541756, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.3626666666666667, "calibration": 73.24318594141187, "cc": 9.783689886391333, "cc_by_type": { "score": 9.783689886391333 }, "cc_tiered": 11.446488784141053, "cc_tiered_open": 14.253192827972441, "cc_tiered_sealed": 8.639784740309665, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 9.783689886391333 } }, "raw_on_A": { "calibration": 60.28751716714082, "capability": 38.880493243091394, "intelligence": 17.47346931904197 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 3.198325647386112, "Emphasis on Accuracy 60:20:20": 2.3288005981136943, "Emphasis on Cost 20:20:60": 3.6607417516659844, "Emphasis on Speed 20:60:20": 4.247790339721065, "Intelligence only": 1.654208859284027 } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "org2ai", "axes": { "calibration": 86.58998638479137, "cost": 46.287007162302075, "intelligence": 51.951877017848766, "speed": 93.07746270925374 }, "calibration": { "parts": { "choice": { "ece_hard": 0.05465848238633132, "ece_score": 89.06830352273374, "mean_tvd": null, "n": 750, "n_ece_hard": 328, "n_tvd": 0, "score": 89.06830352273374 }, "noul": { "abstention_rate": 0.0, "brier": 0.16249185347848963, "ece": 0.09805762809316519, "n": 375, "score": 80.38847438136696 }, "score": { "interval90_coverage": 0.9813333333333333, "interval90_width": 0.6019999999999996, "n": 375, "nrps": 0.1056056821458286, "score": 90.31318125027343, "top_ece": 0.04406534642435141 } }, "score": 86.58998638479137 }, "capability": 69.27093170132007, "composite_ci95": { "A": [ 45.35289667440805, 55.4572699256173 ], "B": [ 42.75715999124745, 53.77505802688884 ], "C": [ 31.495067135005595, 48.32811751847993 ] }, "cost": { "basis": "Self-hosted open weights, no bookable tariff. v1.5-M2 base-model reference rule: base model Qwen/Qwen3.5-4B (model card base_model) -> DeepInfra Qwen/Qwen3.5-4B, USD 0.03/M input and USD 0.15/M output, frozen 25 Sep 2026 price cut-off snapshot (METHOD-v1.5-ADDENDUM-PRICING-INTERPRETATION-1 I-2/I-3; listing deprecated after the cut-off, stays the frozen v1.5 reference). Same basis as the published Wald 4B (v2.0) row. Read 2026-10-10.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.06172102 }, "display": "Wald 4B v2.1", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX PRO 6000 96GB), offline read-only container (--network none); official vllm/vllm-openai:v0.30.0@sha256:8a69ffad... image; author's server wald_serve.native_vision from the model repo, --prompt-format plain (author RUNBOOK's JevBench format), serving.json defaults otherwise (effort none = one pass, noul_decisive_floor 0.81)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 56.69080677700682, "I_sealed": 47.21294725869071, "base": 51.951877017848766, "excess": -0.5953850630236044, "gap": 9.477859518316109, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 70.9521688095628, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 84.43243243243245, "hard": 64.1396665618119, "judge": 76.26895187870798, "standard": 69.8618668899121 } }, "open|noul": { "cc": 66.16425120772948, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 56.52173913043479, "judge": 60.00000000000001, "standard": 77.77777777777777 } }, "open|score": { "cc": 32.9560003137282, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 42.310346490056276, "hard": 31.56850185540615, "judge": 32.18036106421194, "standard": 32.21728301648263 } }, "sealed|choice": { "cc": 55.12375685402749, "n": { "easy": 72, "hard": 260, "judge": 141, "standard": 127 }, "tiers": { "easy": 62.45698195849411, "hard": 49.543805650214345, "judge": 57.07980315559839, "standard": 59.6829772570641 } }, "sealed|noul": { "cc": 53.117408906882595, "n": { "easy": 39, "hard": 132, "judge": 72, "standard": 57 }, "tiers": { "easy": 48.71794871794872, "hard": 50.0, "judge": 55.55555555555556, "standard": 57.89473684210527 } }, "sealed|score": { "cc": 33.39767601516204, "n": { "easy": 33, "hard": 115, "judge": 75, "standard": 77 }, "tiers": { "easy": 38.95455244599806, "hard": 28.60554068871497, "judge": 38.85138751678833, "standard": 32.02294120019875 } } } }, "jevbench_score": 54.28713266716134, "key": "wald-4b-v2-1", "last_measured_on": "2026-10-10", "licence": "Apache-2.0 (weights and server code)", "listing": "ranked", "measured_in": "v1.6.5", "measurement_date_status": "measured", "model_pin": "org2ai/Wald-4B@ce3286f7efd332530e5dccd8a17d6496e029f714", "noul_decisive": { "acc_among_decisive": 0.7813333333333333, "decisive": 375, "decisive_rate": 1.0, "n": 375, "supported": true, "valid": 375 }, "planned_weight_denominator": 1.0, "rank": 2, "ranked": true, "ranks": { "A": 2, "B": 2, "C": 2, "capability": 3 }, "repo": "https://huggingface.co/org2ai/Wald-4B", "scores": { "A": 54.28713266716134, "B": 52.0059503801659, "C": 40.700232429112376 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.20302085168659686, "p50_s_raw": 0.026510425843298435, "p95_s_adjusted": 0.24250074867159127, "p95_s_raw": 0.046250374335795645 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns": { "family": { "calibration": { "accuracy": 0.7586206896551724, "cc": 63.68096433104207, "cc_by_type": { "choice": 75.0, "noul": 66.66666666666667, "score": 49.376226326459516 }, "low_n": true, "n": 29, "n_by_type": { "choice": 16, "noul": 6, "score": 7 }, "score": 63.68096433104207 }, "calibration_gen": { "accuracy": 0.6160714285714286, "cc": 44.10064431077017, "cc_by_type": { "choice": 46.42857142857144, "noul": 61.53846153846154, "score": 24.334899965277536 }, "low_n": false, "n": 112, "n_by_type": { "choice": 56, "noul": 26, "score": 30 }, "score": 44.10064431077017 }, "composite_gen": { "accuracy": 0.6836734693877551, "cc": 57.621281961194256, "cc_by_type": { "choice": 74.07407407407408, "noul": 64.28571428571428, "score": 34.50405752379443 }, "low_n": false, "n": 98, "n_by_type": { "choice": 36, "noul": 28, "score": 34 }, "score": 57.621281961194256 }, "fairness": { "accuracy": 1.0, "cc": 89.90132796385994, "cc_by_type": { "choice": 100.0, "noul": 100.0, "score": 69.70398389157982 }, "low_n": true, "n": 27, "n_by_type": { "choice": 12, "noul": 6, "score": 9 }, "score": 89.90132796385994 }, "jaggedness": { "accuracy": 0.5459459459459459, "cc": 28.653270418657403, "cc_by_type": { "choice": 47.19777674849467, "noul": 12.82051282051282, "score": 25.94152168696473 }, "low_n": false, "n": 185, "n_by_type": { "choice": 97, "noul": 39, "score": 49 }, "score": 28.653270418657403 }, "llm_router": { "accuracy": 0.7424242424242424, "cc": 68.06655796712668, "cc_by_type": { "choice": 56.33999071063632, "noul": 83.56164383561644, "score": 64.29803935512727 }, "low_n": false, "n": 264, "n_by_type": { "choice": 131, "noul": 73, "score": 60 }, "score": 68.06655796712668 }, "long_state": { "accuracy": 0.3793103448275862, "cc": 4.748724836384873, "cc_by_type": { "choice": 40.476190476190474, "noul": -27.27272727272727, "score": 1.0427113056914128 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 11, "score": 7 }, "score": 4.748724836384873 }, "ml_usecase": { "accuracy": 0.5161290322580645, "cc": 22.614498009598538, "cc_by_type": { "choice": 35.38461538461538, "noul": 50.0, "score": -17.541121355819755 }, "low_n": false, "n": 31, "n_by_type": { "choice": 15, "noul": 8, "score": 8 }, "score": 22.614498009598538 }, "multilingual": { "accuracy": 0.6412698412698413, "cc": 46.51177232484513, "cc_by_type": { "choice": 55.217196596506945, "noul": 58.62068965517242, "score": 25.69743072285603 }, "low_n": false, "n": 315, "n_by_type": { "choice": 146, "noul": 87, "score": 82 }, "score": 46.51177232484513 }, "negation_pair_gen": { "accuracy": 0.6964285714285714, "cc": 33.02311941019134, "cc_by_type": { "choice": 61.904761904761905, "noul": 50.0, "score": -12.835403674187873 }, "low_n": false, "n": 56, "n_by_type": { "choice": 28, "noul": 20, "score": 8 }, "score": 33.02311941019134 }, "option_order": { "accuracy": 0.8245614035087719, "cc": 77.80244173140953, "cc_by_type": { "choice": 77.80244173140953 }, "low_n": false, "n": 57, "n_by_type": { "choice": 57 }, "score": 77.80244173140953 }, "perturb_pair": { "accuracy": 0.8666666666666667, "cc": 84.22071661549569, "cc_by_type": { "choice": 76.4089121887287, "noul": 100.0, "score": 76.25323765775835 }, "low_n": false, "n": 60, "n_by_type": { "choice": 34, "noul": 14, "score": 12 }, "score": 84.22071661549569 }, "usecase": { "accuracy": 0.6033755274261603, "cc": 41.10751808936778, "cc_by_type": { "choice": 57.217190210059755, "noul": 43.85964912280702, "score": 22.24571493523657 }, "low_n": false, "n": 237, "n_by_type": { "choice": 111, "noul": 57, "score": 69 }, "score": 41.10751808936778 } }, "lang": { "ar": { "accuracy": 0.8181818181818182, "cc": 42.61103847074362, "cc_by_type": { "choice": 73.33333333333334, "noul": 50.0, "score": 4.49978207889753 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 4, "score": 2 }, "score": 42.61103847074362 }, "cs": { "accuracy": 0.625, "cc": 61.864406779661024, "cc_by_type": { "choice": 23.72881355932204, "noul": 100.0 }, "low_n": true, "n": 8, "n_by_type": { "choice": 5, "noul": 3 }, "score": 61.864406779661024 }, "da": { "accuracy": 0.5416666666666666, "cc": 42.74539505383657, "cc_by_type": { "choice": 28.994082840236683, "noul": 60.00000000000001, "score": 39.24210232127302 }, "low_n": true, "n": 24, "n_by_type": { "choice": 11, "noul": 5, "score": 8 }, "score": 42.74539505383657 }, "de": { "accuracy": 0.7333333333333333, "cc": 55.337376093377294, "cc_by_type": { "choice": 78.27398913699457, "noul": 55.55555555555556, "score": 32.18258358758174 }, "low_n": false, "n": 45, "n_by_type": { "choice": 24, "noul": 9, "score": 12 }, "score": 55.337376093377294 }, "el": { "accuracy": 0.5555555555555556, "cc": 11.934924140935678, "cc_by_type": { "choice": 39.7590361445783, "noul": 60.00000000000001, "score": -63.954263721771284 }, "low_n": true, "n": 18, "n_by_type": { "choice": 11, "noul": 5, "score": 2 }, "score": 11.934924140935678 }, "en": { "accuracy": 0.6657730116175157, "cc": 49.96242193822483, "cc_by_type": { "choice": 59.169938306225426, "noul": 53.38345864661653, "score": 37.33386886183252 }, "low_n": false, "n": 1119, "n_by_type": { "choice": 577, "noul": 266, "score": 276 }, "score": 49.96242193822483 }, "es": { "accuracy": 0.8571428571428571, "cc": 71.78869471940175, "cc_by_type": { "choice": 73.81818181818181, "noul": 100.0, "score": 41.54790234002343 }, "low_n": true, "n": 28, "n_by_type": { "choice": 10, "noul": 11, "score": 7 }, "score": 71.78869471940175 }, "fi": { "accuracy": 0.7777777777777778, "cc": 78.87323943661971, "cc_by_type": { "choice": 57.74647887323943, "noul": 100.0 }, "low_n": true, "n": 9, "n_by_type": { "choice": 6, "noul": 3 }, "score": 78.87323943661971 }, "fr": { "accuracy": 0.7333333333333333, "cc": 53.603908086020034, "cc_by_type": { "choice": 74.61212976022568, "noul": 50.0, "score": 36.19959449783442 }, "low_n": true, "n": 15, "n_by_type": { "choice": 5, "noul": 4, "score": 6 }, "score": 53.603908086020034 }, "hi": { "accuracy": 0.43333333333333335, "cc": 25.416831317917744, "cc_by_type": { "choice": 13.043478260869566, "noul": 60.00000000000001, "score": 3.2070156928836524 }, "low_n": false, "n": 30, "n_by_type": { "choice": 11, "noul": 10, "score": 9 }, "score": 25.416831317917744 }, "id": { "accuracy": 0.6666666666666666, "cc": 33.87096774193548, "cc_by_type": { "choice": 67.74193548387096, "noul": 0.0 }, "low_n": true, "n": 6, "n_by_type": { "choice": 4, "noul": 2 }, "score": 33.87096774193548 }, "it": { "accuracy": 0.7647058823529411, "cc": 54.2758437211133, "cc_by_type": { "choice": 74.61212976022568, "noul": 50.0, "score": 38.21540140311422 }, "low_n": true, "n": 17, "n_by_type": { "choice": 5, "noul": 4, "score": 8 }, "score": 54.2758437211133 }, "ja": { "accuracy": 0.65625, "cc": 49.17418342165371, "cc_by_type": { "choice": 51.21951219512196, "noul": 71.42857142857142, "score": 24.874466641267755 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 49.17418342165371 }, "ko": { "accuracy": 0.5454545454545454, "cc": 55.08222770188039, "cc_by_type": { "choice": 67.21311475409836, "noul": 100.0, "score": -1.9664316484571787 }, "low_n": true, "n": 11, "n_by_type": { "choice": 4, "noul": 2, "score": 5 }, "score": 55.08222770188039 }, "mixed": { "accuracy": 0.9090909090909091, "cc": 82.54649269370202, "cc_by_type": { "choice": 100.0, "noul": 100.0, "score": 47.63947808110607 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 4, "score": 1 }, "score": 82.54649269370202 }, "nl": { "accuracy": 0.38461538461538464, "cc": 9.711332583883976, "cc_by_type": { "choice": -0.2785515320334245, "noul": 33.33333333333333, "score": -3.920784049647974 }, "low_n": true, "n": 13, "n_by_type": { "choice": 5, "noul": 6, "score": 2 }, "score": 9.711332583883976 }, "no": { "accuracy": 0.5714285714285714, "cc": 50.81967213114754, "cc_by_type": { "choice": 1.6393442622950836, "noul": 100.0 }, "low_n": true, "n": 7, "n_by_type": { "choice": 4, "noul": 3 }, "score": 50.81967213114754 }, "pl": { "accuracy": 0.575, "cc": 34.47099050276897, "cc_by_type": { "choice": 76.38828159160472, "noul": 19.999999999999996, "score": 7.024689916702187 }, "low_n": false, "n": 40, "n_by_type": { "choice": 16, "noul": 10, "score": 14 }, "score": 34.47099050276897 }, "pt": { "accuracy": 0.7222222222222222, "cc": 74.79648342566703, "cc_by_type": { "choice": 63.81909547738693, "noul": 100.0, "score": 60.570354799614194 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 6, "score": 5 }, "score": 74.79648342566703 }, "sv": { "accuracy": 0.375, "cc": -8.847329747252749, "cc_by_type": { "choice": -33.333333333333336, "noul": 50.0, "score": -43.20865590842491 }, "low_n": true, "n": 8, "n_by_type": { "choice": 3, "noul": 4, "score": 1 }, "score": 0.0 }, "tr": { "accuracy": 0.7272727272727273, "cc": 39.204683185393755, "cc_by_type": { "choice": 73.6842105263158, "noul": 33.33333333333333, "score": 10.596505696532155 }, "low_n": true, "n": 11, "n_by_type": { "choice": 5, "noul": 3, "score": 3 }, "score": 39.204683185393755 }, "uk": { "accuracy": 0.5454545454545454, "cc": -4.6691240891629535, "cc_by_type": { "choice": 56.52173913043478, "noul": -100.0, "score": 29.47088860207636 }, "low_n": true, "n": 11, "n_by_type": { "choice": 6, "noul": 1, "score": 4 }, "score": 0.0 }, "zh": { "accuracy": 0.75, "cc": 52.01465673807612, "cc_by_type": { "choice": 67.56756756756756, "noul": 100.0, "score": -11.523597353339188 }, "low_n": true, "n": 8, "n_by_type": { "choice": 4, "noul": 3, "score": 1 }, "score": 52.01465673807612 } }, "type": { "choice": { "accuracy": 0.6786666666666666, "cc": 58.36932112627397, "cc_by_type": { "choice": 58.36932112627397 }, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 58.36932112627397 }, "noul": { "accuracy": 0.7813333333333333, "cc": 56.266666666666666, "cc_by_type": { "noul": 56.266666666666666 }, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 56.266666666666666 }, "score": { "accuracy": 0.5066666666666667, "cc": 33.21347273527731, "cc_by_type": { "score": 33.21347273527731 }, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 33.21347273527731 } }, "usecase": { "advertising": { "accuracy": 0.6666666666666666, "cc": 47.43548710573032, "cc_by_type": { "choice": 60.46128500823723, "noul": 60.00000000000001, "score": 21.845176308953697 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 47.43548710573032 }, "code_linting": { "accuracy": 0.5833333333333334, "cc": 41.491312000045305, "cc_by_type": { "choice": 41.08019639934533, "noul": 60.00000000000001, "score": 23.393739600790585 }, "low_n": true, "n": 24, "n_by_type": { "choice": 13, "noul": 5, "score": 6 }, "score": 41.491312000045305 }, "customer_support": { "accuracy": 0.7, "cc": 48.534908440165786, "cc_by_type": { "choice": 51.61290322580645, "noul": 50.0, "score": 43.9918220946909 }, "low_n": true, "n": 20, "n_by_type": { "choice": 8, "noul": 8, "score": 4 }, "score": 48.534908440165786 }, "demand_forecasting": { "accuracy": 0.38461538461538464, "cc": 7.558009820211829, "cc_by_type": { "choice": 39.7590361445783, "noul": 0.0, "score": -17.085006683942815 }, "low_n": true, "n": 26, "n_by_type": { "choice": 11, "noul": 4, "score": 11 }, "score": 7.558009820211829 }, "ecommerce_marketplaces": { "accuracy": 0.6111111111111112, "cc": 60.831849874775266, "cc_by_type": { "choice": 100.0, "noul": 60.00000000000001, "score": 22.495549624325818 }, "low_n": true, "n": 18, "n_by_type": { "choice": 7, "noul": 5, "score": 6 }, "score": 60.831849874775266 }, "feature_extraction": { "accuracy": 0.5517241379310345, "cc": 33.80012553149724, "cc_by_type": { "choice": 48.71794871794871, "noul": 19.999999999999996, "score": 32.68242787654302 }, "low_n": true, "n": 29, "n_by_type": { "choice": 15, "noul": 5, "score": 9 }, "score": 33.80012553149724 }, "financial_crime": { "accuracy": 0.6153846153846154, "cc": 42.38729210014478, "cc_by_type": { "choice": 34.78260869565218, "noul": 100.0, "score": -7.620732395217833 }, "low_n": true, "n": 26, "n_by_type": { "choice": 10, "noul": 8, "score": 8 }, "score": 42.38729210014478 }, "gaming": { "accuracy": 0.5172413793103449, "cc": 29.097480905970304, "cc_by_type": { "choice": 28.712871287128706, "noul": 42.85714285714286, "score": 15.722428573639357 }, "low_n": true, "n": 29, "n_by_type": { "choice": 11, "noul": 7, "score": 11 }, "score": 29.097480905970304 }, "insurance_claims": { "accuracy": 0.5714285714285714, "cc": 59.854855365826985, "cc_by_type": { "choice": 20.634920634920636, "noul": 60.00000000000001, "score": 98.92964546256033 }, "low_n": true, "n": 14, "n_by_type": { "choice": 8, "noul": 5, "score": 1 }, "score": 59.854855365826985 }, "knowledge_graphs": { "accuracy": 0.4782608695652174, "cc": 34.55736075618144, "cc_by_type": { "choice": 26.717557251908392, "noul": 60.00000000000001, "score": 16.954525016635912 }, "low_n": true, "n": 23, "n_by_type": { "choice": 14, "noul": 5, "score": 4 }, "score": 34.55736075618144 }, "lead_generation": { "accuracy": 0.48, "cc": 32.637731364341626, "cc_by_type": { "choice": 83.73983739837398, "noul": 25.0, "score": -10.826643305349105 }, "low_n": true, "n": 25, "n_by_type": { "choice": 8, "noul": 8, "score": 9 }, "score": 32.637731364341626 }, "legal_compliance": { "accuracy": 0.3076923076923077, "cc": -22.12982828671353, "cc_by_type": { "choice": 34.42622950819673, "noul": -100.0, "score": -0.8157143683373169 }, "low_n": true, "n": 13, "n_by_type": { "choice": 8, "noul": 1, "score": 4 }, "score": 0.0 }, "llm_guardrails": { "accuracy": 0.8275862068965517, "cc": 70.67750613389478, "cc_by_type": { "choice": 81.65137614678899, "noul": 75.0, "score": 55.381142254895344 }, "low_n": true, "n": 29, "n_by_type": { "choice": 14, "noul": 8, "score": 7 }, "score": 70.67750613389478 }, "model_routing": { "accuracy": 0.7866666666666666, "cc": 68.49684059479237, "cc_by_type": { "choice": 79.09407665505226, "noul": 86.36363636363636, "score": 40.03280876568852 }, "low_n": false, "n": 150, "n_by_type": { "choice": 74, "noul": 44, "score": 32 }, "score": 68.49684059479237 }, "moderation_trust_safety": { "accuracy": 0.5625, "cc": 32.325588967854976, "cc_by_type": { "choice": 52.63157894736842, "noul": 60.00000000000001, "score": -15.654812043803478 }, "low_n": true, "n": 16, "n_by_type": { "choice": 8, "noul": 5, "score": 3 }, "score": 32.325588967854976 }, "recruiting": { "accuracy": 0.6666666666666666, "cc": 38.753237007491286, "cc_by_type": { "choice": 69.84924623115576, "noul": 33.33333333333333, "score": 13.077131457984759 }, "low_n": true, "n": 21, "n_by_type": { "choice": 9, "noul": 6, "score": 6 }, "score": 38.753237007491286 }, "risk_assessment": { "accuracy": 0.46153846153846156, "cc": 18.588938121855957, "cc_by_type": { "choice": 24.528301886792452, "noul": 33.33333333333333, "score": -2.0948208545578995 }, "low_n": true, "n": 13, "n_by_type": { "choice": 7, "noul": 3, "score": 3 }, "score": 18.588938121855957 }, "scientific_discovery": { "accuracy": 0.375, "cc": 6.8446821135291955, "cc_by_type": { "choice": 28.286852589641434, "noul": -14.28571428571429, "score": 6.532908036660445 }, "low_n": false, "n": 32, "n_by_type": { "choice": 16, "noul": 7, "score": 9 }, "score": 6.8446821135291955 }, "search_retrieval": { "accuracy": 0.8636363636363636, "cc": 76.70730935415533, "cc_by_type": { "choice": 85.68019093078759, "noul": 100.0, "score": 44.44173713167841 }, "low_n": true, "n": 22, "n_by_type": { "choice": 9, "noul": 6, "score": 7 }, "score": 76.70730935415533 } } }, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 80.85810602487076, 87.75388401396593 ], "I": [ 46.645926045990024, 56.00010478358418 ], "capability": [ 64.53882410044396, 71.31542904152454 ], "composite": { "A": [ 45.35289667440805, 55.4572699256173 ], "B": [ 42.75715999124745, 53.77505802688884 ], "C": [ 31.495067135005595, 48.32811751847993 ] }, "seed": 31 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": false, "g_med_used": 10.073244581339713, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 86.58998638479137, "capability": 69.27093170132007, "intelligence": 51.951877017848766 }, "per_type": { "choice": { "accuracy": 0.6786666666666666, "calibration": 89.06830352273374, "cc": 58.36932112627397, "cc_by_type": { "choice": 58.36932112627397 }, "cc_tiered": 63.03796283179515, "cc_tiered_open": 70.9521688095628, "cc_tiered_sealed": 55.12375685402749, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 58.36932112627397 }, "noul": { "accuracy": 0.7813333333333333, "calibration": 80.38847438136696, "cc": 56.266666666666666, "cc_by_type": { "noul": 56.266666666666666 }, "cc_tiered": 59.64083005730603, "cc_tiered_open": 66.16425120772948, "cc_tiered_sealed": 53.117408906882595, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 56.266666666666666 }, "score": { "accuracy": 0.5066666666666667, "calibration": 90.31318125027343, "cc": 33.21347273527731, "cc_by_type": { "score": 33.21347273527731 }, "cc_tiered": 33.176838164445115, "cc_tiered_open": 32.9560003137282, "cc_tiered_sealed": 33.39767601516204, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 33.21347273527731 } }, "raw_on_A": { "calibration": 86.69118099549732, "capability": 69.21287139307924, "intelligence": 51.734561790661175 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 49.82850819150803, "Emphasis on Accuracy 60:20:20": 47.56124181789482, "Emphasis on Cost 20:20:60": 45.197608867054875, "Emphasis on Speed 20:60:20": 58.630700497081925, "Intelligence only": 44.522489120413574 } } ], "tier_weights": { "easy": 0.1, "hard": 0.4, "judge": 0.3, "standard": 0.2 }, "types": { "choice": 0.3333333333333333, "noul": 0.3333333333333333, "score": 0.3333333333333333 }, "v16": { "a2_supplement": null, "baseline_sha256": "279efaa2fef1e8cbae2413adc7a452bf80e04cdc9cd041999a9bbca5bb298074", "capability_formula": "mean(Intelligence, Calibration)", "cost_basis": null, "counts": { "A": 300, "P": 300, "S": 1200, "api_input": 600, "api_smoke_public": 5, "core": 900, "selfhosted_input": 1500 }, "draw_digests": { "A": "a0fe704bdba955ab6f5b847f5c87d4fac801945c17454a73028abdb1682f8de9", "P": "7fa9e2592fcfaf24a959bf56eb247bda8adf60e7b2a1bf7c0a46156189820f68", "S": "66c9d7e43e8db2ca346eee887aee46203dc307fa55d1e7fa28355d7162e97600", "core": "545ebfea24bde76dc92e4068bb4c01b41074375fb0f3b1bf88c9ca9dd95877c7" }, "draw_release": "v1.6-regular-20261010-a2", "equating": { "equating_flag": null, "min_pool": 5, "offsets": { "C": 2.5233026327646186, "I": 0.3106848300439058 }, "pool": [ "blink-v0-3-26b-a4b-nvfp4", "classone-gemma4-e4b", "classone-qwen3.5-2b", "classone-qwen3.5-4b", "clef-omni", "gutsy-0.8b-v0.3", "jiwo-0.8b", "jiwo-4b", "liquid-d1-3b", "liquid-d1-omni-600m", "modernbert-jev-v2", "rsi-jev-v6.1-vl-27b", "standard_one_8b_sh", "vegaml-0.8b", "vegaml-4b", "wald-4b-v2-1" ], "pool_n": 16, "rule": "median over complete full-coverage self-hosted systems of metric(S u P) - metric(A u P), per metric (Intelligence, Calibration); API value = clip(raw_A + offset)" }, "equating_A2": null, "freeze_manifest_sha256": "e100afd2cc1d80819761a344592578cafd0ca8b6a79e9d9c706a562f70b29feb", "g_med_rule": "median gap over ranked self-hosted systems; API systems use the same systems' P-vs-A gap", "offset_bootstrap_B": 0, "ranked_selfhosted": [ "blink-v0-3-26b-a4b-nvfp4", "classone-gemma4-e4b", "classone-qwen3.5-2b", "classone-qwen3.5-4b", "clef-omni", "gutsy-0.8b-v0.3", "jiwo-0.8b", "jiwo-4b", "liquid-d1-3b", "liquid-d1-omni-600m", "modernbert-jev-v2", "rsi-jev-v6.1-vl-27b", "standard_one_8b_sh", "vegaml-0.8b", "vegaml-4b", "wald-4b-v2-1" ], "release": "v1.6-regular-20261010-a2", "scorer_sources_sha256": { "noul_method_v16.py": "119e59486ab3300b972fbcbadb0fe1dceceb124287607375aa61c66063c607d0", "score_v15.py": "0931e05c6410b1b971201834397cb609cdb25f942723c50375632952f6dcc447", "score_v15_headlineA.py": "826f2cbcb57007ec8a161e6eb0111182b8a656c0638027c6c78022a858fd8e2c", "score_v16.py": "02132271117653e6302c509232a581d0094f0e78eac8b1bfc67596fd5d5097e9" }, "scoring_parameters": { "B": 1000, "g_med_fixed": 10.073244581339713, "noul_method": "O1S", "only": null, "seed": 16, "test_allow_partial": false, "workers": 6 }, "seed_commitment": "2926f8a975d7f3f51068ed770b8ed0710aa56121bdb5a4e91119de7dff14fff2", "set_manifest_sha256": { "A": "442364621a9755bd3ea9831c50c720a7ff8d05b1a79dcc0cb69da0cb46bb7f50", "P": "6b2321f8359d87769a95c8e52135542d95ccdc353197e1e5c96abb9b616ce5ba", "S": "8814e4943c5440e7c932111b05b2205acc4019ef45dd0671223bee4c63dd3f01" } }, "views": [ "Balanced 33:33:33", "Emphasis on Accuracy 60:20:20", "Emphasis on Cost 20:20:60", "Emphasis on Speed 20:60:20", "Intelligence only" ] }